github.com/gilgames000/kcc-geth@v1.0.6/core/rawdb/database.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "os" 24 "sync/atomic" 25 "time" 26 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/ethdb" 29 "github.com/ethereum/go-ethereum/ethdb/leveldb" 30 "github.com/ethereum/go-ethereum/ethdb/memorydb" 31 "github.com/ethereum/go-ethereum/log" 32 "github.com/olekukonko/tablewriter" 33 ) 34 35 // freezerdb is a database wrapper that enabled freezer data retrievals. 36 type freezerdb struct { 37 ethdb.KeyValueStore 38 ethdb.AncientStore 39 } 40 41 // Close implements io.Closer, closing both the fast key-value store as well as 42 // the slow ancient tables. 43 func (frdb *freezerdb) Close() error { 44 var errs []error 45 if err := frdb.AncientStore.Close(); err != nil { 46 errs = append(errs, err) 47 } 48 if err := frdb.KeyValueStore.Close(); err != nil { 49 errs = append(errs, err) 50 } 51 if len(errs) != 0 { 52 return fmt.Errorf("%v", errs) 53 } 54 return nil 55 } 56 57 // Freeze is a helper method used for external testing to trigger and block until 58 // a freeze cycle completes, without having to sleep for a minute to trigger the 59 // automatic background run. 60 func (frdb *freezerdb) Freeze(threshold uint64) { 61 // Set the freezer threshold to a temporary value 62 defer func(old uint64) { 63 atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, old) 64 }(atomic.LoadUint64(&frdb.AncientStore.(*freezer).threshold)) 65 atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, threshold) 66 67 // Trigger a freeze cycle and block until it's done 68 trigger := make(chan struct{}, 1) 69 frdb.AncientStore.(*freezer).trigger <- trigger 70 <-trigger 71 } 72 73 // nofreezedb is a database wrapper that disables freezer data retrievals. 74 type nofreezedb struct { 75 ethdb.KeyValueStore 76 } 77 78 // HasAncient returns an error as we don't have a backing chain freezer. 79 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { 80 return false, errNotSupported 81 } 82 83 // Ancient returns an error as we don't have a backing chain freezer. 84 func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { 85 return nil, errNotSupported 86 } 87 88 // Ancients returns an error as we don't have a backing chain freezer. 89 func (db *nofreezedb) Ancients() (uint64, error) { 90 return 0, errNotSupported 91 } 92 93 // AncientSize returns an error as we don't have a backing chain freezer. 94 func (db *nofreezedb) AncientSize(kind string) (uint64, error) { 95 return 0, errNotSupported 96 } 97 98 // AppendAncient returns an error as we don't have a backing chain freezer. 99 func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { 100 return errNotSupported 101 } 102 103 // TruncateAncients returns an error as we don't have a backing chain freezer. 104 func (db *nofreezedb) TruncateAncients(items uint64) error { 105 return errNotSupported 106 } 107 108 // Sync returns an error as we don't have a backing chain freezer. 109 func (db *nofreezedb) Sync() error { 110 return errNotSupported 111 } 112 113 // NewDatabase creates a high level database on top of a given key-value data 114 // store without a freezer moving immutable chain segments into cold storage. 115 func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { 116 return &nofreezedb{ 117 KeyValueStore: db, 118 } 119 } 120 121 // NewDatabaseWithFreezer creates a high level database on top of a given key- 122 // value data store with a freezer moving immutable chain segments into cold 123 // storage. 124 func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { 125 // Create the idle freezer instance 126 frdb, err := newFreezer(freezer, namespace) 127 if err != nil { 128 return nil, err 129 } 130 // Since the freezer can be stored separately from the user's key-value database, 131 // there's a fairly high probability that the user requests invalid combinations 132 // of the freezer and database. Ensure that we don't shoot ourselves in the foot 133 // by serving up conflicting data, leading to both datastores getting corrupted. 134 // 135 // - If both the freezer and key-value store is empty (no genesis), we just 136 // initialized a new empty freezer, so everything's fine. 137 // - If the key-value store is empty, but the freezer is not, we need to make 138 // sure the user's genesis matches the freezer. That will be checked in the 139 // blockchain, since we don't have the genesis block here (nor should we at 140 // this point care, the key-value/freezer combo is valid). 141 // - If neither the key-value store nor the freezer is empty, cross validate 142 // the genesis hashes to make sure they are compatible. If they are, also 143 // ensure that there's no gap between the freezer and sunsequently leveldb. 144 // - If the key-value store is not empty, but the freezer is we might just be 145 // upgrading to the freezer release, or we might have had a small chain and 146 // not frozen anything yet. Ensure that no blocks are missing yet from the 147 // key-value store, since that would mean we already had an old freezer. 148 149 // If the genesis hash is empty, we have a new key-value store, so nothing to 150 // validate in this method. If, however, the genesis hash is not nil, compare 151 // it to the freezer content. 152 if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { 153 if frozen, _ := frdb.Ancients(); frozen > 0 { 154 // If the freezer already contains something, ensure that the genesis blocks 155 // match, otherwise we might mix up freezers across chains and destroy both 156 // the freezer and the key-value store. 157 frgenesis, err := frdb.Ancient(freezerHashTable, 0) 158 if err != nil { 159 return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) 160 } else if !bytes.Equal(kvgenesis, frgenesis) { 161 return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) 162 } 163 // Key-value store and freezer belong to the same network. Ensure that they 164 // are contiguous, otherwise we might end up with a non-functional freezer. 165 if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { 166 // Subsequent header after the freezer limit is missing from the database. 167 // Reject startup is the database has a more recent head. 168 if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { 169 return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) 170 } 171 // Database contains only older data than the freezer, this happens if the 172 // state was wiped and reinited from an existing freezer. 173 } 174 // Otherwise, key-value store continues where the freezer left off, all is fine. 175 // We might have duplicate blocks (crash after freezer write but before key-value 176 // store deletion, but that's fine). 177 } else { 178 // If the freezer is empty, ensure nothing was moved yet from the key-value 179 // store, otherwise we'll end up missing data. We check block #1 to decide 180 // if we froze anything previously or not, but do take care of databases with 181 // only the genesis block. 182 if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { 183 // Key-value store contains more data than the genesis block, make sure we 184 // didn't freeze anything yet. 185 if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { 186 return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") 187 } 188 // Block #1 is still in the database, we're allowed to init a new feezer 189 } 190 // Otherwise, the head header is still the genesis, we're allowed to init a new 191 // feezer. 192 } 193 } 194 // Freezer is consistent with the key-value database, permit combining the two 195 go frdb.freeze(db) 196 197 return &freezerdb{ 198 KeyValueStore: db, 199 AncientStore: frdb, 200 }, nil 201 } 202 203 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a 204 // freezer moving immutable chain segments into cold storage. 205 func NewMemoryDatabase() ethdb.Database { 206 return NewDatabase(memorydb.New()) 207 } 208 209 // NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database 210 // with an initial starting capacity, but without a freezer moving immutable 211 // chain segments into cold storage. 212 func NewMemoryDatabaseWithCap(size int) ethdb.Database { 213 return NewDatabase(memorydb.NewWithCap(size)) 214 } 215 216 // NewLevelDBDatabase creates a persistent key-value database without a freezer 217 // moving immutable chain segments into cold storage. 218 func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { 219 db, err := leveldb.New(file, cache, handles, namespace) 220 if err != nil { 221 return nil, err 222 } 223 return NewDatabase(db), nil 224 } 225 226 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a 227 // freezer moving immutable chain segments into cold storage. 228 func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { 229 kvdb, err := leveldb.New(file, cache, handles, namespace) 230 if err != nil { 231 return nil, err 232 } 233 frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace) 234 if err != nil { 235 kvdb.Close() 236 return nil, err 237 } 238 return frdb, nil 239 } 240 241 type counter uint64 242 243 func (c counter) String() string { 244 return fmt.Sprintf("%d", c) 245 } 246 247 func (c counter) Percentage(current uint64) string { 248 return fmt.Sprintf("%d", current*100/uint64(c)) 249 } 250 251 // stat stores sizes and count for a parameter 252 type stat struct { 253 size common.StorageSize 254 count counter 255 } 256 257 // Add size to the stat and increase the counter by 1 258 func (s *stat) Add(size common.StorageSize) { 259 s.size += size 260 s.count++ 261 } 262 263 func (s *stat) Size() string { 264 return s.size.String() 265 } 266 267 func (s *stat) Count() string { 268 return s.count.String() 269 } 270 271 // InspectDatabase traverses the entire database and checks the size 272 // of all different categories of data. 273 func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { 274 it := db.NewIterator(keyPrefix, keyStart) 275 defer it.Release() 276 277 var ( 278 count int64 279 start = time.Now() 280 logged = time.Now() 281 282 // Key-value store statistics 283 headers stat 284 bodies stat 285 receipts stat 286 tds stat 287 numHashPairings stat 288 hashNumPairings stat 289 tries stat 290 codes stat 291 txLookups stat 292 accountSnaps stat 293 storageSnaps stat 294 preimages stat 295 bloomBits stat 296 cliqueSnaps stat 297 posaSnaps stat 298 299 // Ancient store statistics 300 ancientHeadersSize common.StorageSize 301 ancientBodiesSize common.StorageSize 302 ancientReceiptsSize common.StorageSize 303 ancientTdsSize common.StorageSize 304 ancientHashesSize common.StorageSize 305 306 // Les statistic 307 chtTrieNodes stat 308 bloomTrieNodes stat 309 310 // Meta- and unaccounted data 311 metadata stat 312 unaccounted stat 313 shutdownInfo stat 314 315 // Totals 316 total common.StorageSize 317 ) 318 // Inspect key-value database first. 319 for it.Next() { 320 var ( 321 key = it.Key() 322 size = common.StorageSize(len(key) + len(it.Value())) 323 ) 324 total += size 325 switch { 326 case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): 327 headers.Add(size) 328 case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): 329 bodies.Add(size) 330 case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): 331 receipts.Add(size) 332 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): 333 tds.Add(size) 334 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): 335 numHashPairings.Add(size) 336 case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): 337 hashNumPairings.Add(size) 338 case len(key) == common.HashLength: 339 tries.Add(size) 340 case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: 341 codes.Add(size) 342 case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): 343 txLookups.Add(size) 344 case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): 345 accountSnaps.Add(size) 346 case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): 347 storageSnaps.Add(size) 348 case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): 349 preimages.Add(size) 350 case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): 351 bloomBits.Add(size) 352 case bytes.HasPrefix(key, BloomBitsIndexPrefix): 353 bloomBits.Add(size) 354 case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: 355 cliqueSnaps.Add(size) 356 case bytes.HasPrefix(key, []byte("posa-")) && len(key) == 7+common.HashLength: 357 posaSnaps.Add(size) 358 case bytes.HasPrefix(key, []byte("cht-")) || 359 bytes.HasPrefix(key, []byte("chtIndexV2-")) || 360 bytes.HasPrefix(key, []byte("chtRootV2-")): // Canonical hash trie 361 chtTrieNodes.Add(size) 362 case bytes.HasPrefix(key, []byte("blt-")) || 363 bytes.HasPrefix(key, []byte("bltIndex-")) || 364 bytes.HasPrefix(key, []byte("bltRoot-")): // Bloomtrie sub 365 bloomTrieNodes.Add(size) 366 case bytes.Equal(key, uncleanShutdownKey): 367 shutdownInfo.Add(size) 368 default: 369 var accounted bool 370 for _, meta := range [][]byte{ 371 databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, 372 fastTrieProgressKey, snapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, 373 snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, 374 badBlockKey, 375 } { 376 if bytes.Equal(key, meta) { 377 metadata.Add(size) 378 accounted = true 379 break 380 } 381 } 382 if !accounted { 383 unaccounted.Add(size) 384 } 385 } 386 count++ 387 if count%1000 == 0 && time.Since(logged) > 8*time.Second { 388 log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) 389 logged = time.Now() 390 } 391 } 392 // Inspect append-only file store then. 393 ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} 394 for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { 395 if size, err := db.AncientSize(category); err == nil { 396 *ancientSizes[i] += common.StorageSize(size) 397 total += common.StorageSize(size) 398 } 399 } 400 // Get number of ancient rows inside the freezer 401 ancients := counter(0) 402 if count, err := db.Ancients(); err == nil { 403 ancients = counter(count) 404 } 405 // Display the database statistic. 406 stats := [][]string{ 407 {"Key-Value store", "Headers", headers.Size(), headers.Count()}, 408 {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, 409 {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, 410 {"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, 411 {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, 412 {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, 413 {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, 414 {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, 415 {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, 416 {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, 417 {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, 418 {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, 419 {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, 420 {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, 421 {"Key-Value store", "POSA snapshots", posaSnaps.Size(), posaSnaps.Count()}, 422 {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, 423 {"Key-Value store", "Shutdown metadata", shutdownInfo.Size(), shutdownInfo.Count()}, 424 {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, 425 {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()}, 426 {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()}, 427 {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()}, 428 {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()}, 429 {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, 430 {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, 431 } 432 table := tablewriter.NewWriter(os.Stdout) 433 table.SetHeader([]string{"Database", "Category", "Size", "Items"}) 434 table.SetFooter([]string{"", "Total", total.String(), " "}) 435 table.AppendBulk(stats) 436 table.Render() 437 438 if unaccounted.size > 0 { 439 log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) 440 } 441 442 return nil 443 }