gitlab.com/yannislg/go-pulse@v0.0.0-20210722055913-a3e24e95638d/core/rawdb/database.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "os" 24 "time" 25 26 "github.com/ethereum/go-ethereum/common" 27 "github.com/ethereum/go-ethereum/ethdb" 28 "github.com/ethereum/go-ethereum/ethdb/leveldb" 29 "github.com/ethereum/go-ethereum/ethdb/memorydb" 30 "github.com/ethereum/go-ethereum/log" 31 "github.com/olekukonko/tablewriter" 32 ) 33 34 // freezerdb is a database wrapper that enabled freezer data retrievals. 35 type freezerdb struct { 36 ethdb.KeyValueStore 37 ethdb.AncientStore 38 } 39 40 // Close implements io.Closer, closing both the fast key-value store as well as 41 // the slow ancient tables. 42 func (frdb *freezerdb) Close() error { 43 var errs []error 44 if err := frdb.KeyValueStore.Close(); err != nil { 45 errs = append(errs, err) 46 } 47 if err := frdb.AncientStore.Close(); err != nil { 48 errs = append(errs, err) 49 } 50 if len(errs) != 0 { 51 return fmt.Errorf("%v", errs) 52 } 53 return nil 54 } 55 56 // nofreezedb is a database wrapper that disables freezer data retrievals. 57 type nofreezedb struct { 58 ethdb.KeyValueStore 59 } 60 61 // HasAncient returns an error as we don't have a backing chain freezer. 62 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { 63 return false, errNotSupported 64 } 65 66 // Ancient returns an error as we don't have a backing chain freezer. 67 func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { 68 return nil, errNotSupported 69 } 70 71 // Ancients returns an error as we don't have a backing chain freezer. 72 func (db *nofreezedb) Ancients() (uint64, error) { 73 return 0, errNotSupported 74 } 75 76 // AncientSize returns an error as we don't have a backing chain freezer. 77 func (db *nofreezedb) AncientSize(kind string) (uint64, error) { 78 return 0, errNotSupported 79 } 80 81 // AppendAncient returns an error as we don't have a backing chain freezer. 82 func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { 83 return errNotSupported 84 } 85 86 // TruncateAncients returns an error as we don't have a backing chain freezer. 87 func (db *nofreezedb) TruncateAncients(items uint64) error { 88 return errNotSupported 89 } 90 91 // Sync returns an error as we don't have a backing chain freezer. 92 func (db *nofreezedb) Sync() error { 93 return errNotSupported 94 } 95 96 // NewDatabase creates a high level database on top of a given key-value data 97 // store without a freezer moving immutable chain segments into cold storage. 98 func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { 99 return &nofreezedb{ 100 KeyValueStore: db, 101 } 102 } 103 104 // NewDatabaseWithFreezer creates a high level database on top of a given key- 105 // value data store with a freezer moving immutable chain segments into cold 106 // storage. 107 func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { 108 // Create the idle freezer instance 109 frdb, err := newFreezer(freezer, namespace) 110 if err != nil { 111 return nil, err 112 } 113 // Since the freezer can be stored separately from the user's key-value database, 114 // there's a fairly high probability that the user requests invalid combinations 115 // of the freezer and database. Ensure that we don't shoot ourselves in the foot 116 // by serving up conflicting data, leading to both datastores getting corrupted. 117 // 118 // - If both the freezer and key-value store is empty (no genesis), we just 119 // initialized a new empty freezer, so everything's fine. 120 // - If the key-value store is empty, but the freezer is not, we need to make 121 // sure the user's genesis matches the freezer. That will be checked in the 122 // blockchain, since we don't have the genesis block here (nor should we at 123 // this point care, the key-value/freezer combo is valid). 124 // - If neither the key-value store nor the freezer is empty, cross validate 125 // the genesis hashes to make sure they are compatible. If they are, also 126 // ensure that there's no gap between the freezer and sunsequently leveldb. 127 // - If the key-value store is not empty, but the freezer is we might just be 128 // upgrading to the freezer release, or we might have had a small chain and 129 // not frozen anything yet. Ensure that no blocks are missing yet from the 130 // key-value store, since that would mean we already had an old freezer. 131 132 // If the genesis hash is empty, we have a new key-value store, so nothing to 133 // validate in this method. If, however, the genesis hash is not nil, compare 134 // it to the freezer content. 135 if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { 136 if frozen, _ := frdb.Ancients(); frozen > 0 { 137 // If the freezer already contains something, ensure that the genesis blocks 138 // match, otherwise we might mix up freezers across chains and destroy both 139 // the freezer and the key-value store. 140 if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { 141 return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) 142 } 143 // Key-value store and freezer belong to the same network. Ensure that they 144 // are contiguous, otherwise we might end up with a non-functional freezer. 145 if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { 146 // Subsequent header after the freezer limit is missing from the database. 147 // Reject startup is the database has a more recent head. 148 if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { 149 return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) 150 } 151 // Database contains only older data than the freezer, this happens if the 152 // state was wiped and reinited from an existing freezer. 153 } 154 // Otherwise, key-value store continues where the freezer left off, all is fine. 155 // We might have duplicate blocks (crash after freezer write but before key-value 156 // store deletion, but that's fine). 157 } else { 158 // If the freezer is empty, ensure nothing was moved yet from the key-value 159 // store, otherwise we'll end up missing data. We check block #1 to decide 160 // if we froze anything previously or not, but do take care of databases with 161 // only the genesis block. 162 if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { 163 // Key-value store contains more data than the genesis block, make sure we 164 // didn't freeze anything yet. 165 if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { 166 return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") 167 } 168 // Block #1 is still in the database, we're allowed to init a new feezer 169 } 170 // Otherwise, the head header is still the genesis, we're allowed to init a new 171 // feezer. 172 } 173 } 174 // Freezer is consistent with the key-value database, permit combining the two 175 go frdb.freeze(db) 176 177 return &freezerdb{ 178 KeyValueStore: db, 179 AncientStore: frdb, 180 }, nil 181 } 182 183 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a 184 // freezer moving immutable chain segments into cold storage. 185 func NewMemoryDatabase() ethdb.Database { 186 return NewDatabase(memorydb.New()) 187 } 188 189 // NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database 190 // with an initial starting capacity, but without a freezer moving immutable 191 // chain segments into cold storage. 192 func NewMemoryDatabaseWithCap(size int) ethdb.Database { 193 return NewDatabase(memorydb.NewWithCap(size)) 194 } 195 196 // NewLevelDBDatabase creates a persistent key-value database without a freezer 197 // moving immutable chain segments into cold storage. 198 func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { 199 db, err := leveldb.New(file, cache, handles, namespace) 200 if err != nil { 201 return nil, err 202 } 203 return NewDatabase(db), nil 204 } 205 206 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a 207 // freezer moving immutable chain segments into cold storage. 208 func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { 209 kvdb, err := leveldb.New(file, cache, handles, namespace) 210 if err != nil { 211 return nil, err 212 } 213 frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace) 214 if err != nil { 215 kvdb.Close() 216 return nil, err 217 } 218 return frdb, nil 219 } 220 221 // InspectDatabase traverses the entire database and checks the size 222 // of all different categories of data. 223 func InspectDatabase(db ethdb.Database) error { 224 it := db.NewIterator(nil, nil) 225 defer it.Release() 226 227 var ( 228 count int64 229 start = time.Now() 230 logged = time.Now() 231 232 // Key-value store statistics 233 total common.StorageSize 234 headerSize common.StorageSize 235 bodySize common.StorageSize 236 receiptSize common.StorageSize 237 tdSize common.StorageSize 238 numHashPairing common.StorageSize 239 hashNumPairing common.StorageSize 240 trieSize common.StorageSize 241 txlookupSize common.StorageSize 242 accountSnapSize common.StorageSize 243 storageSnapSize common.StorageSize 244 preimageSize common.StorageSize 245 bloomBitsSize common.StorageSize 246 cliqueSnapsSize common.StorageSize 247 parliaSnapsSize common.StorageSize 248 249 // Ancient store statistics 250 ancientHeaders common.StorageSize 251 ancientBodies common.StorageSize 252 ancientReceipts common.StorageSize 253 ancientHashes common.StorageSize 254 ancientTds common.StorageSize 255 256 // Les statistic 257 chtTrieNodes common.StorageSize 258 bloomTrieNodes common.StorageSize 259 260 // Meta- and unaccounted data 261 metadata common.StorageSize 262 unaccounted common.StorageSize 263 ) 264 // Inspect key-value database first. 265 for it.Next() { 266 var ( 267 key = it.Key() 268 size = common.StorageSize(len(key) + len(it.Value())) 269 ) 270 total += size 271 switch { 272 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): 273 tdSize += size 274 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): 275 numHashPairing += size 276 case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): 277 headerSize += size 278 case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): 279 hashNumPairing += size 280 case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): 281 bodySize += size 282 case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): 283 receiptSize += size 284 case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): 285 txlookupSize += size 286 case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): 287 accountSnapSize += size 288 case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): 289 storageSnapSize += size 290 case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): 291 preimageSize += size 292 case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): 293 bloomBitsSize += size 294 case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: 295 cliqueSnapsSize += size 296 case bytes.HasPrefix(key, []byte("parlia-")) && len(key) == 7+common.HashLength: 297 parliaSnapsSize += size 298 case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: 299 chtTrieNodes += size 300 case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: 301 bloomTrieNodes += size 302 case len(key) == common.HashLength: 303 trieSize += size 304 default: 305 var accounted bool 306 for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { 307 if bytes.Equal(key, meta) { 308 metadata += size 309 accounted = true 310 break 311 } 312 } 313 if !accounted { 314 unaccounted += size 315 } 316 } 317 count += 1 318 if count%1000 == 0 && time.Since(logged) > 8*time.Second { 319 log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) 320 logged = time.Now() 321 } 322 } 323 // Inspect append-only file store then. 324 ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} 325 for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { 326 if size, err := db.AncientSize(category); err == nil { 327 *ancients[i] += common.StorageSize(size) 328 total += common.StorageSize(size) 329 } 330 } 331 // Display the database statistic. 332 stats := [][]string{ 333 {"Key-Value store", "Headers", headerSize.String()}, 334 {"Key-Value store", "Bodies", bodySize.String()}, 335 {"Key-Value store", "Receipts", receiptSize.String()}, 336 {"Key-Value store", "Difficulties", tdSize.String()}, 337 {"Key-Value store", "Block number->hash", numHashPairing.String()}, 338 {"Key-Value store", "Block hash->number", hashNumPairing.String()}, 339 {"Key-Value store", "Transaction index", txlookupSize.String()}, 340 {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, 341 {"Key-Value store", "Trie nodes", trieSize.String()}, 342 {"Key-Value store", "Trie preimages", preimageSize.String()}, 343 {"Key-Value store", "Account snapshot", accountSnapSize.String()}, 344 {"Key-Value store", "Storage snapshot", storageSnapSize.String()}, 345 {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, 346 {"Key-Value store", "Parlia snapshots", parliaSnapsSize.String()}, 347 {"Key-Value store", "Singleton metadata", metadata.String()}, 348 {"Ancient store", "Headers", ancientHeaders.String()}, 349 {"Ancient store", "Bodies", ancientBodies.String()}, 350 {"Ancient store", "Receipts", ancientReceipts.String()}, 351 {"Ancient store", "Difficulties", ancientTds.String()}, 352 {"Ancient store", "Block number->hash", ancientHashes.String()}, 353 {"Light client", "CHT trie nodes", chtTrieNodes.String()}, 354 {"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, 355 } 356 table := tablewriter.NewWriter(os.Stdout) 357 table.SetHeader([]string{"Database", "Category", "Size"}) 358 table.SetFooter([]string{"", "Total", total.String()}) 359 table.AppendBulk(stats) 360 table.Render() 361 362 if unaccounted > 0 { 363 log.Error("Database contains unaccounted data", "size", unaccounted) 364 } 365 return nil 366 }