github.com/bigzoro/my_simplechain@v0.0.0-20240315012955-8ad0a2a29bb9/core/rawdb/database.go (about) 1 // Copyright 2018 The go-simplechain Authors 2 // This file is part of the go-simplechain library. 3 // 4 // The go-simplechain library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-simplechain library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-simplechain library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "os" 24 "time" 25 26 "github.com/bigzoro/my_simplechain/common" 27 "github.com/bigzoro/my_simplechain/ethdb" 28 "github.com/bigzoro/my_simplechain/ethdb/leveldb" 29 "github.com/bigzoro/my_simplechain/ethdb/memorydb" 30 "github.com/bigzoro/my_simplechain/log" 31 "github.com/olekukonko/tablewriter" 32 ) 33 34 // freezerdb is a database wrapper that enabled freezer data retrievals. 35 type freezerdb struct { 36 ethdb.KeyValueStore 37 ethdb.AncientStore 38 } 39 40 // Close implements io.Closer, closing both the fast key-value store as well as 41 // the slow ancient tables. 42 func (frdb *freezerdb) Close() error { 43 var errs []error 44 if err := frdb.KeyValueStore.Close(); err != nil { 45 errs = append(errs, err) 46 } 47 if err := frdb.AncientStore.Close(); err != nil { 48 errs = append(errs, err) 49 } 50 if len(errs) != 0 { 51 return fmt.Errorf("%v", errs) 52 } 53 return nil 54 } 55 56 // nofreezedb is a database wrapper that disables freezer data retrievals. 57 type nofreezedb struct { 58 ethdb.KeyValueStore 59 } 60 61 // HasAncient returns an error as we don't have a backing chain freezer. 62 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { 63 return false, errNotSupported 64 } 65 66 // Ancient returns an error as we don't have a backing chain freezer. 67 func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { 68 return nil, errNotSupported 69 } 70 71 // Ancients returns an error as we don't have a backing chain freezer. 72 func (db *nofreezedb) Ancients() (uint64, error) { 73 return 0, errNotSupported 74 } 75 76 // AncientSize returns an error as we don't have a backing chain freezer. 77 func (db *nofreezedb) AncientSize(kind string) (uint64, error) { 78 return 0, errNotSupported 79 } 80 81 // AppendAncient returns an error as we don't have a backing chain freezer. 82 func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { 83 return errNotSupported 84 } 85 86 // TruncateAncients returns an error as we don't have a backing chain freezer. 87 func (db *nofreezedb) TruncateAncients(items uint64) error { 88 return errNotSupported 89 } 90 91 // Sync returns an error as we don't have a backing chain freezer. 92 func (db *nofreezedb) Sync() error { 93 return errNotSupported 94 } 95 96 // NewDatabase creates a high level database on top of a given key-value data 97 // store without a freezer moving immutable chain segments into cold storage. 98 func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { 99 return &nofreezedb{ 100 KeyValueStore: db, 101 } 102 } 103 104 // NewDatabaseWithFreezer creates a high level database on top of a given key- 105 // value data store with a freezer moving immutable chain segments into cold 106 // storage. 107 func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { 108 // Create the idle freezer instance 109 frdb, err := newFreezer(freezer, namespace) 110 if err != nil { 111 return nil, err 112 } 113 // Since the freezer can be stored separately from the user's key-value database, 114 // there's a fairly high probability that the user requests invalid combinations 115 // of the freezer and database. Ensure that we don't shoot ourselves in the foot 116 // by serving up conflicting data, leading to both datastores getting corrupted. 117 // 118 // - If both the freezer and key-value store is empty (no genesis), we just 119 // initialized a new empty freezer, so everything's fine. 120 // - If the key-value store is empty, but the freezer is not, we need to make 121 // sure the user's genesis matches the freezer. That will be checked in the 122 // blockchain, since we don't have the genesis block here (nor should we at 123 // this point care, the key-value/freezer combo is valid). 124 // - If neither the key-value store nor the freezer is empty, cross validate 125 // the genesis hashes to make sure they are compatible. If they are, also 126 // ensure that there's no gap between the freezer and sunsequently leveldb. 127 // - If the key-value store is not empty, but the freezer is we might just be 128 // upgrading to the freezer release, or we might have had a small chain and 129 // not frozen anything yet. Ensure that no blocks are missing yet from the 130 // key-value store, since that would mean we already had an old freezer. 131 132 // If the genesis hash is empty, we have a new key-value store, so nothing to 133 // validate in this method. If, however, the genesis hash is not nil, compare 134 // it to the freezer content. 135 if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { 136 if frozen, _ := frdb.Ancients(); frozen > 0 { 137 // If the freezer already contains something, ensure that the genesis blocks 138 // match, otherwise we might mix up freezers across chains and destroy both 139 // the freezer and the key-value store. 140 if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { 141 return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) 142 } 143 // Key-value store and freezer belong to the same network. Ensure that they 144 // are contiguous, otherwise we might end up with a non-functional freezer. 145 if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { 146 // Subsequent header after the freezer limit is missing from the database. 147 // Reject startup is the database has a more recent head. 148 if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { 149 return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) 150 } 151 // Database contains only older data than the freezer, this happens if the 152 // state was wiped and reinited from an existing freezer. 153 } 154 // Otherwise, key-value store continues where the freezer left off, all is fine. 155 // We might have duplicate blocks (crash after freezer write but before key-value 156 // store deletion, but that's fine). 157 } else { 158 // If the freezer is empty, ensure nothing was moved yet from the key-value 159 // store, otherwise we'll end up missing data. We check block #1 to decide 160 // if we froze anything previously or not, but do take care of databases with 161 // only the genesis block. 162 if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { 163 // Key-value store contains more data than the genesis block, make sure we 164 // didn't freeze anything yet. 165 if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { 166 return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") 167 } 168 // Block #1 is still in the database, we're allowed to init a new feezer 169 } 170 // Otherwise, the head header is still the genesis, we're allowed to init a new 171 // feezer. 172 } 173 } 174 // Freezer is consistent with the key-value database, permit combining the two 175 go frdb.freeze(db) 176 177 return &freezerdb{ 178 KeyValueStore: db, 179 AncientStore: frdb, 180 }, nil 181 } 182 183 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a 184 // freezer moving immutable chain segments into cold storage. 185 func NewMemoryDatabase() ethdb.Database { 186 return NewDatabase(memorydb.New()) 187 } 188 189 // NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database 190 // with an initial starting capacity, but without a freezer moving immutable 191 // chain segments into cold storage. 192 func NewMemoryDatabaseWithCap(size int) ethdb.Database { 193 return NewDatabase(memorydb.NewWithCap(size)) 194 } 195 196 // NewLevelDBDatabase creates a persistent key-value database without a freezer 197 // moving immutable chain segments into cold storage. 198 func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) { 199 db, err := leveldb.New(file, cache, handles, namespace) 200 if err != nil { 201 return nil, err 202 } 203 return NewDatabase(db), nil 204 } 205 206 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a 207 // freezer moving immutable chain segments into cold storage. 208 func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { 209 kvdb, err := leveldb.New(file, cache, handles, namespace) 210 if err != nil { 211 return nil, err 212 } 213 frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace) 214 if err != nil { 215 kvdb.Close() 216 return nil, err 217 } 218 return frdb, nil 219 } 220 221 // InspectDatabase traverses the entire database and checks the size 222 // of all different categories of data. 223 func InspectDatabase(db ethdb.Database) error { 224 it := db.NewIterator() 225 defer it.Release() 226 227 var ( 228 count int64 229 start = time.Now() 230 logged = time.Now() 231 232 // Key-value store statistics 233 total common.StorageSize 234 headerSize common.StorageSize 235 bodySize common.StorageSize 236 receiptSize common.StorageSize 237 tdSize common.StorageSize 238 numHashPairing common.StorageSize 239 hashNumPairing common.StorageSize 240 trieSize common.StorageSize 241 txlookupSize common.StorageSize 242 preimageSize common.StorageSize 243 bloomBitsSize common.StorageSize 244 cliqueSnapsSize common.StorageSize 245 246 // Ancient store statistics 247 ancientHeaders common.StorageSize 248 ancientBodies common.StorageSize 249 ancientReceipts common.StorageSize 250 ancientHashes common.StorageSize 251 ancientTds common.StorageSize 252 253 // Les statistic 254 chtTrieNodes common.StorageSize 255 bloomTrieNodes common.StorageSize 256 257 // Meta- and unaccounted data 258 metadata common.StorageSize 259 unaccounted common.StorageSize 260 ) 261 // Inspect key-value database first. 262 for it.Next() { 263 var ( 264 key = it.Key() 265 size = common.StorageSize(len(key) + len(it.Value())) 266 ) 267 total += size 268 switch { 269 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): 270 tdSize += size 271 case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): 272 numHashPairing += size 273 case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): 274 headerSize += size 275 case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): 276 hashNumPairing += size 277 case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): 278 bodySize += size 279 case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): 280 receiptSize += size 281 case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): 282 txlookupSize += size 283 case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): 284 preimageSize += size 285 case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): 286 bloomBitsSize += size 287 case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: 288 cliqueSnapsSize += size 289 case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: 290 chtTrieNodes += size 291 case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: 292 bloomTrieNodes += size 293 case len(key) == common.HashLength: 294 trieSize += size 295 default: 296 var accounted bool 297 for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { 298 if bytes.Equal(key, meta) { 299 metadata += size 300 accounted = true 301 break 302 } 303 } 304 if !accounted { 305 unaccounted += size 306 } 307 } 308 count += 1 309 if count%1000 == 0 && time.Since(logged) > 8*time.Second { 310 log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) 311 logged = time.Now() 312 } 313 } 314 // Inspect append-only file store then. 315 ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} 316 for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { 317 if size, err := db.AncientSize(category); err == nil { 318 *ancients[i] += common.StorageSize(size) 319 total += common.StorageSize(size) 320 } 321 } 322 // Display the database statistic. 323 stats := [][]string{ 324 {"Key-Value store", "Headers", headerSize.String()}, 325 {"Key-Value store", "Bodies", bodySize.String()}, 326 {"Key-Value store", "Receipts", receiptSize.String()}, 327 {"Key-Value store", "Difficulties", tdSize.String()}, 328 {"Key-Value store", "Block number->hash", numHashPairing.String()}, 329 {"Key-Value store", "Block hash->number", hashNumPairing.String()}, 330 {"Key-Value store", "Transaction index", txlookupSize.String()}, 331 {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, 332 {"Key-Value store", "Trie nodes", trieSize.String()}, 333 {"Key-Value store", "Trie preimages", preimageSize.String()}, 334 {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, 335 {"Key-Value store", "Singleton metadata", metadata.String()}, 336 {"Ancient store", "Headers", ancientHeaders.String()}, 337 {"Ancient store", "Bodies", ancientBodies.String()}, 338 {"Ancient store", "Receipts", ancientReceipts.String()}, 339 {"Ancient store", "Difficulties", ancientTds.String()}, 340 {"Ancient store", "Block number->hash", ancientHashes.String()}, 341 {"Light client", "CHT trie nodes", chtTrieNodes.String()}, 342 {"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, 343 } 344 table := tablewriter.NewWriter(os.Stdout) 345 table.SetHeader([]string{"Database", "Category", "Size"}) 346 table.SetFooter([]string{"", "Total", total.String()}) 347 table.AppendBulk(stats) 348 table.Render() 349 350 if unaccounted > 0 { 351 log.Error("Database contains unaccounted data", "size", unaccounted) 352 } 353 return nil 354 }