github.com/MetalBlockchain/subnet-evm@v0.6.3/core/rawdb/database.go (about) 1 // (c) 2019-2020, Ava Labs, Inc. 2 // 3 // This file is a derived work, based on the go-ethereum library whose original 4 // notices appear below. 5 // 6 // It is distributed under a license compatible with the licensing terms of the 7 // original code from which it is derived. 8 // 9 // Much love to the original authors for their work. 10 // ********** 11 // Copyright 2018 The go-ethereum Authors 12 // This file is part of the go-ethereum library. 13 // 14 // The go-ethereum library is free software: you can redistribute it and/or modify 15 // it under the terms of the GNU Lesser General Public License as published by 16 // the Free Software Foundation, either version 3 of the License, or 17 // (at your option) any later version. 18 // 19 // The go-ethereum library is distributed in the hope that it will be useful, 20 // but WITHOUT ANY WARRANTY; without even the implied warranty of 21 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 // GNU Lesser General Public License for more details. 23 // 24 // You should have received a copy of the GNU Lesser General Public License 25 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 26 27 package rawdb 28 29 import ( 30 "bytes" 31 "errors" 32 "fmt" 33 "os" 34 "path/filepath" 35 "time" 36 37 "github.com/ethereum/go-ethereum/common" 38 "github.com/ethereum/go-ethereum/ethdb" 39 "github.com/ethereum/go-ethereum/ethdb/leveldb" 40 "github.com/ethereum/go-ethereum/ethdb/memorydb" 41 "github.com/ethereum/go-ethereum/log" 42 "github.com/olekukonko/tablewriter" 43 ) 44 45 // nofreezedb is a database wrapper that disables freezer data retrievals. 46 type nofreezedb struct { 47 ethdb.KeyValueStore 48 } 49 50 // HasAncient returns an error as we don't have a backing chain freezer. 51 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { 52 return false, errNotSupported 53 } 54 55 // Ancient returns an error as we don't have a backing chain freezer. 56 func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { 57 return nil, errNotSupported 58 } 59 60 // AncientRange returns an error as we don't have a backing chain freezer. 61 func (db *nofreezedb) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) { 62 return nil, errNotSupported 63 } 64 65 // Ancients returns an error as we don't have a backing chain freezer. 66 func (db *nofreezedb) Ancients() (uint64, error) { 67 return 0, errNotSupported 68 } 69 70 // Tail returns an error as we don't have a backing chain freezer. 71 func (db *nofreezedb) Tail() (uint64, error) { 72 return 0, errNotSupported 73 } 74 75 // AncientSize returns an error as we don't have a backing chain freezer. 76 func (db *nofreezedb) AncientSize(kind string) (uint64, error) { 77 return 0, errNotSupported 78 } 79 80 // ModifyAncients is not supported. 81 func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { 82 return 0, errNotSupported 83 } 84 85 // TruncateHead returns an error as we don't have a backing chain freezer. 86 func (db *nofreezedb) TruncateHead(items uint64) (uint64, error) { 87 return 0, errNotSupported 88 } 89 90 // TruncateTail returns an error as we don't have a backing chain freezer. 91 func (db *nofreezedb) TruncateTail(items uint64) (uint64, error) { 92 return 0, errNotSupported 93 } 94 95 // Sync returns an error as we don't have a backing chain freezer. 96 func (db *nofreezedb) Sync() error { 97 return errNotSupported 98 } 99 100 func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { 101 // Unlike other ancient-related methods, this method does not return 102 // errNotSupported when invoked. 103 // The reason for this is that the caller might want to do several things: 104 // 1. Check if something is in freezer, 105 // 2. If not, check leveldb. 106 // 107 // This will work, since the ancient-checks inside 'fn' will return errors, 108 // and the leveldb work will continue. 109 // 110 // If we instead were to return errNotSupported here, then the caller would 111 // have to explicitly check for that, having an extra clause to do the 112 // non-ancient operations. 113 return fn(db) 114 } 115 116 // MigrateTable processes the entries in a given table in sequence 117 // converting them to a new format if they're of an old format. 118 func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { 119 return errNotSupported 120 } 121 122 // AncientDatadir returns an error as we don't have a backing chain freezer. 123 func (db *nofreezedb) AncientDatadir() (string, error) { 124 return "", errNotSupported 125 } 126 127 // NewDatabase creates a high level database on top of a given key-value data 128 // store without a freezer moving immutable chain segments into cold storage. 129 func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { 130 return &nofreezedb{KeyValueStore: db} 131 } 132 133 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a 134 // freezer moving immutable chain segments into cold storage. 135 func NewMemoryDatabase() ethdb.Database { 136 return NewDatabase(memorydb.New()) 137 } 138 139 // NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database 140 // with an initial starting capacity, but without a freezer moving immutable 141 // chain segments into cold storage. 142 func NewMemoryDatabaseWithCap(size int) ethdb.Database { 143 return NewDatabase(memorydb.NewWithCap(size)) 144 } 145 146 // NewLevelDBDatabase creates a persistent key-value database without a freezer 147 // moving immutable chain segments into cold storage. 148 func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { 149 db, err := leveldb.New(file, cache, handles, namespace, readonly) 150 if err != nil { 151 return nil, err 152 } 153 log.Info("Using LevelDB as the backing database") 154 return NewDatabase(db), nil 155 } 156 157 const ( 158 dbPebble = "pebble" 159 dbLeveldb = "leveldb" 160 ) 161 162 // hasPreexistingDb checks the given data directory whether a database is already 163 // instantiated at that location, and if so, returns the type of database (or the 164 // empty string). 165 func hasPreexistingDb(path string) string { 166 if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { 167 return "" // No pre-existing db 168 } 169 if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil { 170 if err != nil { 171 panic(err) // only possible if the pattern is malformed 172 } 173 return dbPebble 174 } 175 return dbLeveldb 176 } 177 178 // OpenOptions contains the options to apply when opening a database. 179 // OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used. 180 type OpenOptions struct { 181 Type string // "leveldb" | "pebble" 182 Directory string // the datadir 183 Namespace string // the namespace for database relevant metrics 184 Cache int // the capacity(in megabytes) of the data caching 185 Handles int // number of files to be open simultaneously 186 ReadOnly bool 187 } 188 189 // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. 190 // 191 // type == null type != null 192 // +---------------------------------------- 193 // db is non-existent | pebble default | specified type 194 // db is existent | from db | specified type (if compatible) 195 func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { 196 // Reject any unsupported database type 197 if len(o.Type) != 0 && o.Type != dbLeveldb && o.Type != dbPebble { 198 return nil, fmt.Errorf("unknown db.engine %v", o.Type) 199 } 200 // Retrieve any pre-existing database's type and use that or the requested one 201 // as long as there's no conflict between the two types 202 existingDb := hasPreexistingDb(o.Directory) 203 if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { 204 return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) 205 } 206 if o.Type == dbPebble || existingDb == dbPebble { 207 if PebbleEnabled { 208 log.Info("Using pebble as the backing database") 209 return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) 210 } else { 211 return nil, errors.New("db.engine 'pebble' not supported on this platform") 212 } 213 } 214 if o.Type == dbLeveldb || existingDb == dbLeveldb { 215 log.Info("Using leveldb as the backing database") 216 return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) 217 } 218 // No pre-existing database, no user-requested one either. Default to Pebble 219 // on supported platforms and LevelDB on anything else. 220 if PebbleEnabled { 221 log.Info("Defaulting to pebble as the backing database") 222 return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) 223 } else { 224 log.Info("Defaulting to leveldb as the backing database") 225 return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) 226 } 227 } 228 229 // Open opens both a disk-based key-value database such as leveldb or pebble, but also 230 // integrates it with a freezer database -- if the AncientDir option has been 231 // set on the provided OpenOptions. 232 // The passed o.AncientDir indicates the path of root ancient directory where 233 // the chain freezer can be opened. 234 func Open(o OpenOptions) (ethdb.Database, error) { 235 kvdb, err := openKeyValueDatabase(o) 236 if err != nil { 237 return nil, err 238 } 239 return kvdb, nil 240 } 241 242 type counter uint64 243 244 func (c counter) String() string { 245 return fmt.Sprintf("%d", c) 246 } 247 248 func (c counter) Percentage(current uint64) string { 249 return fmt.Sprintf("%d", current*100/uint64(c)) 250 } 251 252 // stat stores sizes and count for a parameter 253 type stat struct { 254 size common.StorageSize 255 count counter 256 } 257 258 // Add size to the stat and increase the counter by 1 259 func (s *stat) Add(size common.StorageSize) { 260 s.size += size 261 s.count++ 262 } 263 264 func (s *stat) Size() string { 265 return s.size.String() 266 } 267 268 func (s *stat) Count() string { 269 return s.count.String() 270 } 271 272 // InspectDatabase traverses the entire database and checks the size 273 // of all different categories of data. 274 func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { 275 it := db.NewIterator(keyPrefix, keyStart) 276 defer it.Release() 277 278 var ( 279 count int64 280 start = time.Now() 281 logged = time.Now() 282 283 // Key-value store statistics 284 headers stat 285 bodies stat 286 receipts stat 287 numHashPairings stat 288 hashNumPairings stat 289 tries stat 290 codes stat 291 txLookups stat 292 accountSnaps stat 293 storageSnaps stat 294 preimages stat 295 bloomBits stat 296 cliqueSnaps stat 297 298 // State sync statistics 299 codeToFetch stat 300 syncProgress stat 301 syncSegments stat 302 syncPerformed stat 303 304 // Les statistic 305 chtTrieNodes stat 306 bloomTrieNodes stat 307 308 // Meta- and unaccounted data 309 metadata stat 310 unaccounted stat 311 312 // Totals 313 total common.StorageSize 314 ) 315 // Inspect key-value database first. 316 for it.Next() { 317 var ( 318 key = it.Key() 319 size = common.StorageSize(len(key) + len(it.Value())) 320 ) 321 total += size 322 switch { 323 case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): 324 headers.Add(size) 325 case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): 326 bodies.Add(size) 327 case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): 328 receipts.Add(size) 329 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): 330 numHashPairings.Add(size) 331 case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): 332 hashNumPairings.Add(size) 333 case len(key) == common.HashLength: 334 tries.Add(size) 335 case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: 336 codes.Add(size) 337 case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): 338 txLookups.Add(size) 339 case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): 340 accountSnaps.Add(size) 341 case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): 342 storageSnaps.Add(size) 343 case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): 344 preimages.Add(size) 345 case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): 346 metadata.Add(size) 347 case bytes.HasPrefix(key, upgradeConfigPrefix) && len(key) == (len(upgradeConfigPrefix)+common.HashLength): 348 metadata.Add(size) 349 case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): 350 bloomBits.Add(size) 351 case bytes.HasPrefix(key, BloomBitsIndexPrefix): 352 bloomBits.Add(size) 353 case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength: 354 syncProgress.Add(size) 355 case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength: 356 syncSegments.Add(size) 357 case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength: 358 codeToFetch.Add(size) 359 case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength: 360 syncPerformed.Add(size) 361 default: 362 var accounted bool 363 for _, meta := range [][]byte{ 364 databaseVersionKey, headHeaderKey, headBlockKey, 365 snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey, 366 uncleanShutdownKey, syncRootKey, txIndexTailKey, 367 } { 368 if bytes.Equal(key, meta) { 369 metadata.Add(size) 370 accounted = true 371 break 372 } 373 } 374 if !accounted { 375 unaccounted.Add(size) 376 } 377 } 378 count++ 379 if count%1000 == 0 && time.Since(logged) > 8*time.Second { 380 log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) 381 logged = time.Now() 382 } 383 } 384 // Display the database statistic. 385 stats := [][]string{ 386 {"Key-Value store", "Headers", headers.Size(), headers.Count()}, 387 {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, 388 {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, 389 {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, 390 {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, 391 {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, 392 {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, 393 {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, 394 {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, 395 {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, 396 {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, 397 {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, 398 {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, 399 {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, 400 {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, 401 {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, 402 {"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()}, 403 {"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()}, 404 {"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()}, 405 {"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()}, 406 } 407 table := tablewriter.NewWriter(os.Stdout) 408 table.SetHeader([]string{"Database", "Category", "Size", "Items"}) 409 table.SetFooter([]string{"", "Total", total.String(), " "}) 410 table.AppendBulk(stats) 411 table.Render() 412 413 if unaccounted.size > 0 { 414 log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) 415 } 416 return nil 417 } 418 419 // ClearPrefix removes all keys in db that begin with prefix and match an 420 // expected key length. [keyLen] should include the length of the prefix. 421 func ClearPrefix(db ethdb.KeyValueStore, prefix []byte, keyLen int) error { 422 it := db.NewIterator(prefix, nil) 423 defer it.Release() 424 425 batch := db.NewBatch() 426 for it.Next() { 427 key := common.CopyBytes(it.Key()) 428 if len(key) != keyLen { 429 // avoid deleting keys that do not match the expected length 430 continue 431 } 432 if err := batch.Delete(key); err != nil { 433 return err 434 } 435 if batch.ValueSize() > ethdb.IdealBatchSize { 436 if err := batch.Write(); err != nil { 437 return err 438 } 439 batch.Reset() 440 } 441 } 442 if err := it.Error(); err != nil { 443 return err 444 } 445 return batch.Write() 446 }