github.com/bcskill/bcschain/v3@v3.4.9-beta2/light/postprocess.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package light 18 19 import ( 20 "encoding/binary" 21 "errors" 22 "math/big" 23 "time" 24 25 "github.com/bcskill/bcschain/v3/common" 26 "github.com/bcskill/bcschain/v3/common/bitutil" 27 "github.com/bcskill/bcschain/v3/core" 28 "github.com/bcskill/bcschain/v3/core/rawdb" 29 "github.com/bcskill/bcschain/v3/core/types" 30 "github.com/bcskill/bcschain/v3/log" 31 "github.com/bcskill/bcschain/v3/params" 32 "github.com/bcskill/bcschain/v3/rlp" 33 "github.com/bcskill/bcschain/v3/trie" 34 ) 35 36 const ( 37 // CHTFrequencyClient is the block frequency for creating CHTs on the client side. 38 CHTFrequencyClient = 32768 39 40 // CHTFrequencyServer is the block frequency for creating CHTs on the server side. 41 // Eventually this can be merged back with the client version, but that requires a 42 // full database upgrade, so that should be left for a suitable moment. 43 CHTFrequencyServer = 4096 44 45 HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available 46 HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated 47 ) 48 49 // trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with 50 // the appropriate section index and head hash. It is used to start light syncing from this checkpoint 51 // and avoid downloading the entire header chain while still being able to securely access old headers/logs. 52 type trustedCheckpoint struct { 53 name string 54 sectionIdx uint64 55 sectionHead, chtRoot, bloomTrieRoot common.Hash 56 } 57 58 var ( 59 mainnetCheckpoint = trustedCheckpoint{ 60 name: "mainnet", 61 sectionIdx: 153, 62 sectionHead: common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"), 63 chtRoot: common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"), 64 bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"), 65 } 66 67 ropstenCheckpoint = trustedCheckpoint{ 68 name: "ropsten", 69 sectionIdx: 79, 70 sectionHead: common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"), 71 chtRoot: common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"), 72 bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"), 73 } 74 ) 75 76 // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to 77 var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ 78 params.MainnetGenesisHash: mainnetCheckpoint, 79 params.TestnetGenesisHash: ropstenCheckpoint, 80 } 81 82 var ( 83 ErrNoTrustedCht = errors.New("No trusted canonical hash trie") 84 ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") 85 ErrNoHeader = errors.New("Header not found") 86 chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash 87 ChtTablePrefix = "cht-" 88 ) 89 90 // ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format 91 type ChtNode struct { 92 Hash common.Hash 93 Td *big.Int 94 } 95 96 // GetChtRoot reads the CHT root assoctiated to the given section from the database 97 // Note that sectionIdx is specified according to LES/1 CHT section size 98 func GetChtRoot(db common.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 99 var encNumber [8]byte 100 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 101 data, _ := db.GlobalTable().Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...)) 102 return common.BytesToHash(data) 103 } 104 105 // GetChtV2Root reads the CHT root assoctiated to the given section from the database 106 // Note that sectionIdx is specified according to LES/2 CHT section size 107 func GetChtV2Root(db common.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 108 return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) 109 } 110 111 // StoreChtRoot writes the CHT root assoctiated to the given section into the database 112 // Note that sectionIdx is specified according to LES/1 CHT section size 113 func StoreChtRoot(db common.Database, sectionIdx uint64, sectionHead, root common.Hash) { 114 var encNumber [8]byte 115 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 116 if err := db.GlobalTable().Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()); err != nil { 117 log.Error("Cannot store cht root", "err", err) 118 } 119 } 120 121 // ChtIndexerBackend implements core.ChainIndexerBackend 122 type ChtIndexerBackend struct { 123 diskdb common.Database 124 triedb *trie.Database 125 section, sectionSize uint64 126 lastHash common.Hash 127 trie *trie.Trie 128 } 129 130 // NewBloomTrieIndexer creates a BloomTrie chain indexer 131 func NewChtIndexer(db common.Database, clientMode bool) *core.ChainIndexer { 132 var sectionSize, confirmReq uint64 133 if clientMode { 134 sectionSize = CHTFrequencyClient 135 confirmReq = HelperTrieConfirmations 136 } else { 137 sectionSize = CHTFrequencyServer 138 confirmReq = HelperTrieProcessConfirmations 139 } 140 idb := common.NewTablePrefixer(db.GlobalTable(), "chtIndex-") 141 backend := &ChtIndexerBackend{ 142 diskdb: db, 143 triedb: trie.NewDatabase(common.NewTablePrefixer(db.GlobalTable(), ChtTablePrefix)), 144 sectionSize: sectionSize, 145 } 146 return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") 147 } 148 149 // Reset implements core.ChainIndexerBackend 150 func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 151 var root common.Hash 152 if section > 0 { 153 root = GetChtRoot(c.diskdb, section-1, lastSectionHead) 154 } 155 var err error 156 c.trie, err = trie.New(root, c.triedb) 157 c.section = section 158 return err 159 } 160 161 // Process implements core.ChainIndexerBackend 162 func (c *ChtIndexerBackend) Process(header *types.Header) { 163 hash, num := header.Hash(), header.Number.Uint64() 164 c.lastHash = hash 165 166 td := rawdb.ReadTd(c.diskdb.GlobalTable(), hash, num) 167 if td == nil { 168 panic(nil) 169 } 170 var encNumber [8]byte 171 binary.BigEndian.PutUint64(encNumber[:], num) 172 data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) 173 c.trie.Update(encNumber[:], data) 174 } 175 176 // Commit implements core.ChainIndexerBackend 177 func (c *ChtIndexerBackend) Commit() error { 178 root, err := c.trie.Commit(nil) 179 if err != nil { 180 return err 181 } 182 if err := c.triedb.Commit(root, false); err != nil { 183 log.Error("Cannot commit cht indexer backend", "err", err) 184 } 185 186 if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { 187 log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) 188 } 189 StoreChtRoot(c.diskdb, c.section, c.lastHash, root) 190 return nil 191 } 192 193 const ( 194 BloomTrieFrequency = 32768 195 ethBloomBitsSection = 4096 196 ethBloomBitsConfirmations = 256 197 ) 198 199 var ( 200 bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash 201 BloomTrieTablePrefix = "blt-" 202 ) 203 204 // GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database 205 func GetBloomTrieRoot(db common.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 206 var encNumber [8]byte 207 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 208 data, _ := db.GlobalTable().Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) 209 return common.BytesToHash(data) 210 } 211 212 // StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database 213 func StoreBloomTrieRoot(db common.Database, sectionIdx uint64, sectionHead, root common.Hash) { 214 var encNumber [8]byte 215 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 216 if err := db.GlobalTable().Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()); err != nil { 217 log.Error("Cannot store bloom trie root", "err", err) 218 } 219 } 220 221 // BloomTrieIndexerBackend implements core.ChainIndexerBackend 222 type BloomTrieIndexerBackend struct { 223 diskdb common.Database 224 triedb *trie.Database 225 section, parentSectionSize, bloomTrieRatio uint64 226 trie *trie.Trie 227 sectionHeads []common.Hash 228 } 229 230 // NewBloomTrieIndexer creates a BloomTrie chain indexer 231 func NewBloomTrieIndexer(db common.Database, clientMode bool) *core.ChainIndexer { 232 backend := &BloomTrieIndexerBackend{ 233 diskdb: db, 234 triedb: trie.NewDatabase(common.NewTablePrefixer(db.GlobalTable(), BloomTrieTablePrefix)), 235 } 236 idb := common.NewTablePrefixer(db.GlobalTable(), "bltIndex-") 237 238 var confirmReq uint64 239 if clientMode { 240 backend.parentSectionSize = BloomTrieFrequency 241 confirmReq = HelperTrieConfirmations 242 } else { 243 backend.parentSectionSize = ethBloomBitsSection 244 confirmReq = HelperTrieProcessConfirmations 245 } 246 backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize 247 backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) 248 return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") 249 } 250 251 // Reset implements core.ChainIndexerBackend 252 func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 253 var root common.Hash 254 if section > 0 { 255 root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) 256 } 257 var err error 258 b.trie, err = trie.New(root, b.triedb) 259 b.section = section 260 return err 261 } 262 263 // Process implements core.ChainIndexerBackend 264 func (b *BloomTrieIndexerBackend) Process(header *types.Header) { 265 num := header.Number.Uint64() - b.section*BloomTrieFrequency 266 if (num+1)%b.parentSectionSize == 0 { 267 b.sectionHeads[num/b.parentSectionSize] = header.Hash() 268 } 269 } 270 271 // Commit implements core.ChainIndexerBackend 272 func (b *BloomTrieIndexerBackend) Commit() error { 273 var compSize, decompSize uint64 274 275 for i := uint(0); i < types.BloomBitLength; i++ { 276 var encKey [10]byte 277 binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) 278 binary.BigEndian.PutUint64(encKey[2:10], b.section) 279 var decomp []byte 280 for j := uint64(0); j < b.bloomTrieRatio; j++ { 281 data := rawdb.ReadBloomBits(b.diskdb.GlobalTable(), i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) 282 decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) 283 if err2 != nil { 284 return err2 285 } 286 decomp = append(decomp, decompData...) 287 } 288 comp := bitutil.CompressBytes(decomp) 289 290 decompSize += uint64(len(decomp)) 291 compSize += uint64(len(comp)) 292 if len(comp) > 0 { 293 b.trie.Update(encKey[:], comp) 294 } else { 295 b.trie.Delete(encKey[:]) 296 } 297 } 298 root, err := b.trie.Commit(nil) 299 if err != nil { 300 return err 301 } 302 if err := b.triedb.Commit(root, false); err != nil { 303 log.Error("Cannot store bloom trie indexer trie db", "err", err) 304 } 305 306 sectionHead := b.sectionHeads[b.bloomTrieRatio-1] 307 log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) 308 StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) 309 310 return nil 311 }