github.com/zhiqiangxu/go-ethereum@v1.9.16-0.20210824055606-be91cfdebc48/core/rawdb/database.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "os" 24 "time" 25 26 "github.com/olekukonko/tablewriter" 27 "github.com/zhiqiangxu/go-ethereum/common" 28 "github.com/zhiqiangxu/go-ethereum/ethdb" 29 "github.com/zhiqiangxu/go-ethereum/ethdb/leveldb" 30 "github.com/zhiqiangxu/go-ethereum/ethdb/memorydb" 31 "github.com/zhiqiangxu/go-ethereum/log" 32 ) 33 34 // freezerdb is a database wrapper that enabled freezer data retrievals. 35 type freezerdb struct { 36 ethdb.KeyValueStore 37 ethdb.AncientStore 38 } 39 40 // Close implements io.Closer, closing both the fast key-value store as well as 41 // the slow ancient tables. 42 func (frdb *freezerdb) Close() error { 43 var errs []error 44 if err := frdb.AncientStore.Close(); err != nil { 45 errs = append(errs, err) 46 } 47 if err := frdb.KeyValueStore.Close(); err != nil { 48 errs = append(errs, err) 49 } 50 if len(errs) != 0 { 51 return fmt.Errorf("%v", errs) 52 } 53 return nil 54 } 55 56 // nofreezedb is a database wrapper that disables freezer data retrievals. 57 type nofreezedb struct { 58 ethdb.KeyValueStore 59 } 60 61 // HasAncient returns an error as we don't have a backing chain freezer. 62 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { 63 return false, errNotSupported 64 } 65 66 // Ancient returns an error as we don't have a backing chain freezer. 67 func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { 68 return nil, errNotSupported 69 } 70 71 // Ancients returns an error as we don't have a backing chain freezer. 72 func (db *nofreezedb) Ancients() (uint64, error) { 73 return 0, errNotSupported 74 } 75 76 // AncientSize returns an error as we don't have a backing chain freezer. 77 func (db *nofreezedb) AncientSize(kind string) (uint64, error) { 78 return 0, errNotSupported 79 } 80 81 // AppendAncient returns an error as we don't have a backing chain freezer. 82 func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { 83 return errNotSupported 84 } 85 86 // TruncateAncients returns an error as we don't have a backing chain freezer. 87 func (db *nofreezedb) TruncateAncients(items uint64) error { 88 return errNotSupported 89 } 90 91 // Sync returns an error as we don't have a backing chain freezer. 92 func (db *nofreezedb) Sync() error { 93 return errNotSupported 94 } 95 96 // NewDatabase creates a high level database on top of a given key-value data 97 // store without a freezer moving immutable chain segments into cold storage. 98 func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { 99 return &nofreezedb{ 100 KeyValueStore: db, 101 } 102 } 103 104 // NewDatabaseWithFreezer creates a high level database on top of a given key- 105 // value data store with a freezer moving immutable chain segments into cold 106 // storage. 107 func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { 108 // Create the idle freezer instance 109 frdb, err := newFreezer(freezer, namespace) 110 if err != nil { 111 return nil, err 112 } 113 // Since the freezer can be stored separately from the user's key-value database, 114 // there's a fairly high probability that the user requests invalid combinations 115 // of the freezer and database. Ensure that we don't shoot ourselves in the foot 116 // by serving up conflicting data, leading to both datastores getting corrupted. 117 // 118 // - If both the freezer and key-value store is empty (no genesis), we just 119 // initialized a new empty freezer, so everything's fine. 120 // - If the key-value store is empty, but the freezer is not, we need to make 121 // sure the user's genesis matches the freezer. That will be checked in the 122 // blockchain, since we don't have the genesis block here (nor should we at 123 // this point care, the key-value/freezer combo is valid). 124 // - If neither the key-value store nor the freezer is empty, cross validate 125 // the genesis hashes to make sure they are compatible. If they are, also 126 // ensure that there's no gap between the freezer and sunsequently leveldb. 127 // - If the key-value store is not empty, but the freezer is we might just be 128 // upgrading to the freezer release, or we might have had a small chain and 129 // not frozen anything yet. Ensure that no blocks are missing yet from the 130 // key-value store, since that would mean we already had an old freezer. 131 132 // If the genesis hash is empty, we have a new key-value store, so nothing to 133 // validate in this method. If, however, the genesis hash is not nil, compare 134 // it to the freezer content. 135 if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { 136 if frozen, _ := frdb.Ancients(); frozen > 0 { 137 // If the freezer already contains something, ensure that the genesis blocks 138 // match, otherwise we might mix up freezers across chains and destroy both 139 // the freezer and the key-value store. 140 if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { 141 return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) 142 } 143 // Key-value store and freezer belong to the same network. Ensure that they 144 // are contiguous, otherwise we might end up with a non-functional freezer. 145 if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { 146 // Subsequent header after the freezer limit is missing from the database. 147 // Reject startup is the database has a more recent head. 148 if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { 149 return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) 150 } 151 // Database contains only older data than the freezer, this happens if the 152 // state was wiped and reinited from an existing freezer. 153 } 154 // Otherwise, key-value store continues where the freezer left off, all is fine. 155 // We might have duplicate blocks (crash after freezer write but before key-value 156 // store deletion, but that's fine). 157 } else { 158 // If the freezer is empty, ensure nothing was moved yet from the key-value 159 // store, otherwise we'll end up missing data. We check block #1 to decide 160 // if we froze anything previously or not, but do take care of databases with 161 // only the genesis block. 162 if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { 163 // Key-value store contains more data than the genesis block, make sure we 164 // didn't freeze anything yet. 165 if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { 166 return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") 167 } 168 // Block #1 is still in the database, we're allowed to init a new feezer 169 } 170 // Otherwise, the head header is still the genesis, we're allowed to init a new 171 // feezer. 172 } 173 } 174 // Freezer is consistent with the key-value database, permit combining the two 175 go frdb.freeze(db) 176 177 return &freezerdb{ 178 KeyValueStore: db, 179 AncientStore: frdb, 180 }, nil 181 } 182 183 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a 184 // freezer moving immutable chain segments into cold storage. 185 func NewMemoryDatabase() ethdb.Database { 186 return NewDatabase(memorydb.New()) 187 } 188 189 // NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database 190 // with an initial starting capacity, but without a freezer moving immutable 191 // chain segments into cold storage. 192 func NewMemoryDatabaseWithCap(size int) ethdb.Database { 193 return NewDatabase(memorydb.NewWithCap(size)) 194 } 195 196 // NewLevelDBDatabase creates a persistent key-value database without a freezer 197 // moving immutable chain segments into cold storage. 198 func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { 199 db, err := leveldb.New(file, cache, handles, namespace) 200 if err != nil { 201 return nil, err 202 } 203 return NewDatabase(db), nil 204 } 205 206 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a 207 // freezer moving immutable chain segments into cold storage. 208 func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { 209 kvdb, err := leveldb.New(file, cache, handles, namespace) 210 if err != nil { 211 return nil, err 212 } 213 frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace) 214 if err != nil { 215 kvdb.Close() 216 return nil, err 217 } 218 return frdb, nil 219 } 220 221 // InspectDatabase traverses the entire database and checks the size 222 // of all different categories of data. 223 func InspectDatabase(db ethdb.Database) error { 224 it := db.NewIterator(nil, nil) 225 defer it.Release() 226 227 var ( 228 count int64 229 start = time.Now() 230 logged = time.Now() 231 232 // Key-value store statistics 233 total common.StorageSize 234 headerSize common.StorageSize 235 bodySize common.StorageSize 236 receiptSize common.StorageSize 237 tdSize common.StorageSize 238 numHashPairing common.StorageSize 239 hashNumPairing common.StorageSize 240 trieSize common.StorageSize 241 txlookupSize common.StorageSize 242 accountSnapSize common.StorageSize 243 storageSnapSize common.StorageSize 244 preimageSize common.StorageSize 245 bloomBitsSize common.StorageSize 246 cliqueSnapsSize common.StorageSize 247 248 // Ancient store statistics 249 ancientHeaders common.StorageSize 250 ancientBodies common.StorageSize 251 ancientReceipts common.StorageSize 252 ancientHashes common.StorageSize 253 ancientTds common.StorageSize 254 255 // Les statistic 256 chtTrieNodes common.StorageSize 257 bloomTrieNodes common.StorageSize 258 259 // Meta- and unaccounted data 260 metadata common.StorageSize 261 unaccounted common.StorageSize 262 ) 263 // Inspect key-value database first. 264 for it.Next() { 265 var ( 266 key = it.Key() 267 size = common.StorageSize(len(key) + len(it.Value())) 268 ) 269 total += size 270 switch { 271 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): 272 tdSize += size 273 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): 274 numHashPairing += size 275 case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): 276 headerSize += size 277 case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): 278 hashNumPairing += size 279 case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): 280 bodySize += size 281 case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): 282 receiptSize += size 283 case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): 284 txlookupSize += size 285 case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): 286 accountSnapSize += size 287 case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): 288 storageSnapSize += size 289 case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): 290 preimageSize += size 291 case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): 292 bloomBitsSize += size 293 case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: 294 cliqueSnapsSize += size 295 case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: 296 chtTrieNodes += size 297 case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: 298 bloomTrieNodes += size 299 case len(key) == common.HashLength: 300 trieSize += size 301 default: 302 var accounted bool 303 for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { 304 if bytes.Equal(key, meta) { 305 metadata += size 306 accounted = true 307 break 308 } 309 } 310 if !accounted { 311 unaccounted += size 312 } 313 } 314 count += 1 315 if count%1000 == 0 && time.Since(logged) > 8*time.Second { 316 log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) 317 logged = time.Now() 318 } 319 } 320 // Inspect append-only file store then. 321 ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} 322 for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { 323 if size, err := db.AncientSize(category); err == nil { 324 *ancients[i] += common.StorageSize(size) 325 total += common.StorageSize(size) 326 } 327 } 328 // Display the database statistic. 329 stats := [][]string{ 330 {"Key-Value store", "Headers", headerSize.String()}, 331 {"Key-Value store", "Bodies", bodySize.String()}, 332 {"Key-Value store", "Receipts", receiptSize.String()}, 333 {"Key-Value store", "Difficulties", tdSize.String()}, 334 {"Key-Value store", "Block number->hash", numHashPairing.String()}, 335 {"Key-Value store", "Block hash->number", hashNumPairing.String()}, 336 {"Key-Value store", "Transaction index", txlookupSize.String()}, 337 {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, 338 {"Key-Value store", "Trie nodes", trieSize.String()}, 339 {"Key-Value store", "Trie preimages", preimageSize.String()}, 340 {"Key-Value store", "Account snapshot", accountSnapSize.String()}, 341 {"Key-Value store", "Storage snapshot", storageSnapSize.String()}, 342 {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, 343 {"Key-Value store", "Singleton metadata", metadata.String()}, 344 {"Ancient store", "Headers", ancientHeaders.String()}, 345 {"Ancient store", "Bodies", ancientBodies.String()}, 346 {"Ancient store", "Receipts", ancientReceipts.String()}, 347 {"Ancient store", "Difficulties", ancientTds.String()}, 348 {"Ancient store", "Block number->hash", ancientHashes.String()}, 349 {"Light client", "CHT trie nodes", chtTrieNodes.String()}, 350 {"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, 351 } 352 table := tablewriter.NewWriter(os.Stdout) 353 table.SetHeader([]string{"Database", "Category", "Size"}) 354 table.SetFooter([]string{"", "Total", total.String()}) 355 table.AppendBulk(stats) 356 table.Render() 357 358 if unaccounted > 0 { 359 log.Error("Database contains unaccounted data", "size", unaccounted) 360 } 361 return nil 362 }