github.com/Elemental-core/elementalcore@v0.0.0-20191206075037-63891242267a/light/postprocess.go (about) 1 // Copyright 2016 The elementalcore Authors 2 // This file is part of the elementalcore library. 3 // 4 // The elementalcore library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The elementalcore library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the elementalcore library. If not, see <http://www.gnu.org/licenses/>. 16 17 package light 18 19 import ( 20 "encoding/binary" 21 "errors" 22 "fmt" 23 "math/big" 24 "time" 25 26 "github.com/Elemental-core/elementalcore/common" 27 "github.com/Elemental-core/elementalcore/common/bitutil" 28 "github.com/Elemental-core/elementalcore/core" 29 "github.com/Elemental-core/elementalcore/core/types" 30 "github.com/Elemental-core/elementalcore/ethdb" 31 "github.com/Elemental-core/elementalcore/log" 32 "github.com/Elemental-core/elementalcore/params" 33 "github.com/Elemental-core/elementalcore/rlp" 34 "github.com/Elemental-core/elementalcore/trie" 35 ) 36 37 const ( 38 ChtFrequency = 32768 39 ChtV1Frequency = 4096 // as long as we want to retain LES/1 compatibility, servers generate CHTs with the old, higher frequency 40 HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available 41 HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated 42 ) 43 44 // trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with 45 // the appropriate section index and head hash. It is used to start light syncing from this checkpoint 46 // and avoid downloading the entire header chain while still being able to securely access old headers/logs. 47 type trustedCheckpoint struct { 48 name string 49 sectionIdx uint64 50 sectionHead, chtRoot, bloomTrieRoot common.Hash 51 } 52 53 var ( 54 mainnetCheckpoint = trustedCheckpoint{ 55 name: "ETH mainnet", 56 sectionIdx: 129, 57 sectionHead: common.HexToHash("64100587c8ec9a76870056d07cb0f58622552d16de6253a59cac4b580c899501"), 58 chtRoot: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca"), 59 bloomTrieRoot: common.HexToHash("0db524b2c4a2a9520a42fd842b02d2e8fb58ff37c75cf57bd0eb82daeace6716"), 60 } 61 62 ropstenCheckpoint = trustedCheckpoint{ 63 name: "Ropsten testnet", 64 sectionIdx: 50, 65 sectionHead: common.HexToHash("00bd65923a1aa67f85e6b4ae67835784dd54be165c37f056691723c55bf016bd"), 66 chtRoot: common.HexToHash("6f56dc61936752cc1f8c84b4addabdbe6a1c19693de3f21cb818362df2117f03"), 67 bloomTrieRoot: common.HexToHash("aca7d7c504d22737242effc3fdc604a762a0af9ced898036b5986c3a15220208"), 68 } 69 ) 70 71 // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to 72 var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ 73 params.MainnetGenesisHash: mainnetCheckpoint, 74 } 75 76 var ( 77 ErrNoTrustedCht = errors.New("No trusted canonical hash trie") 78 ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") 79 ErrNoHeader = errors.New("Header not found") 80 chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash 81 ChtTablePrefix = "cht-" 82 ) 83 84 // ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format 85 type ChtNode struct { 86 Hash common.Hash 87 Td *big.Int 88 } 89 90 // GetChtRoot reads the CHT root assoctiated to the given section from the database 91 // Note that sectionIdx is specified according to LES/1 CHT section size 92 func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 93 var encNumber [8]byte 94 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 95 data, _ := db.Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...)) 96 return common.BytesToHash(data) 97 } 98 99 // GetChtV2Root reads the CHT root assoctiated to the given section from the database 100 // Note that sectionIdx is specified according to LES/2 CHT section size 101 func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 102 return GetChtRoot(db, (sectionIdx+1)*(ChtFrequency/ChtV1Frequency)-1, sectionHead) 103 } 104 105 // StoreChtRoot writes the CHT root assoctiated to the given section into the database 106 // Note that sectionIdx is specified according to LES/1 CHT section size 107 func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 108 var encNumber [8]byte 109 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 110 db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 111 } 112 113 // ChtIndexerBackend implements core.ChainIndexerBackend 114 type ChtIndexerBackend struct { 115 db, cdb ethdb.Database 116 section, sectionSize uint64 117 lastHash common.Hash 118 trie *trie.Trie 119 } 120 121 // NewBloomTrieIndexer creates a BloomTrie chain indexer 122 func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 123 cdb := ethdb.NewTable(db, ChtTablePrefix) 124 idb := ethdb.NewTable(db, "chtIndex-") 125 var sectionSize, confirmReq uint64 126 if clientMode { 127 sectionSize = ChtFrequency 128 confirmReq = HelperTrieConfirmations 129 } else { 130 sectionSize = ChtV1Frequency 131 confirmReq = HelperTrieProcessConfirmations 132 } 133 return core.NewChainIndexer(db, idb, &ChtIndexerBackend{db: db, cdb: cdb, sectionSize: sectionSize}, sectionSize, confirmReq, time.Millisecond*100, "cht") 134 } 135 136 // Reset implements core.ChainIndexerBackend 137 func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 138 var root common.Hash 139 if section > 0 { 140 root = GetChtRoot(c.db, section-1, lastSectionHead) 141 } 142 var err error 143 c.trie, err = trie.New(root, c.cdb) 144 c.section = section 145 return err 146 } 147 148 // Process implements core.ChainIndexerBackend 149 func (c *ChtIndexerBackend) Process(header *types.Header) { 150 hash, num := header.Hash(), header.Number.Uint64() 151 c.lastHash = hash 152 153 td := core.GetTd(c.db, hash, num) 154 if td == nil { 155 panic(nil) 156 } 157 var encNumber [8]byte 158 binary.BigEndian.PutUint64(encNumber[:], num) 159 data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) 160 c.trie.Update(encNumber[:], data) 161 } 162 163 // Commit implements core.ChainIndexerBackend 164 func (c *ChtIndexerBackend) Commit() error { 165 batch := c.cdb.NewBatch() 166 root, err := c.trie.CommitTo(batch) 167 if err != nil { 168 return err 169 } else { 170 batch.Write() 171 if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 { 172 log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) 173 } 174 StoreChtRoot(c.db, c.section, c.lastHash, root) 175 } 176 return nil 177 } 178 179 const ( 180 BloomTrieFrequency = 32768 181 ethBloomBitsSection = 4096 182 ethBloomBitsConfirmations = 256 183 ) 184 185 var ( 186 bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash 187 BloomTrieTablePrefix = "blt-" 188 ) 189 190 // GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database 191 func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 192 var encNumber [8]byte 193 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 194 data, _ := db.Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) 195 return common.BytesToHash(data) 196 } 197 198 // StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database 199 func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 200 var encNumber [8]byte 201 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 202 db.Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 203 } 204 205 // BloomTrieIndexerBackend implements core.ChainIndexerBackend 206 type BloomTrieIndexerBackend struct { 207 db, cdb ethdb.Database 208 section, parentSectionSize, bloomTrieRatio uint64 209 trie *trie.Trie 210 sectionHeads []common.Hash 211 } 212 213 // NewBloomTrieIndexer creates a BloomTrie chain indexer 214 func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 215 cdb := ethdb.NewTable(db, BloomTrieTablePrefix) 216 idb := ethdb.NewTable(db, "bltIndex-") 217 backend := &BloomTrieIndexerBackend{db: db, cdb: cdb} 218 var confirmReq uint64 219 if clientMode { 220 backend.parentSectionSize = BloomTrieFrequency 221 confirmReq = HelperTrieConfirmations 222 } else { 223 backend.parentSectionSize = ethBloomBitsSection 224 confirmReq = HelperTrieProcessConfirmations 225 } 226 backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize 227 backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) 228 return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") 229 } 230 231 // Reset implements core.ChainIndexerBackend 232 func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 233 var root common.Hash 234 if section > 0 { 235 root = GetBloomTrieRoot(b.db, section-1, lastSectionHead) 236 } 237 var err error 238 b.trie, err = trie.New(root, b.cdb) 239 b.section = section 240 return err 241 } 242 243 // Process implements core.ChainIndexerBackend 244 func (b *BloomTrieIndexerBackend) Process(header *types.Header) { 245 num := header.Number.Uint64() - b.section*BloomTrieFrequency 246 if (num+1)%b.parentSectionSize == 0 { 247 b.sectionHeads[num/b.parentSectionSize] = header.Hash() 248 } 249 } 250 251 // Commit implements core.ChainIndexerBackend 252 func (b *BloomTrieIndexerBackend) Commit() error { 253 var compSize, decompSize uint64 254 255 for i := uint(0); i < types.BloomBitLength; i++ { 256 var encKey [10]byte 257 binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) 258 binary.BigEndian.PutUint64(encKey[2:10], b.section) 259 var decomp []byte 260 for j := uint64(0); j < b.bloomTrieRatio; j++ { 261 data, err := core.GetBloomBits(b.db, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) 262 if err != nil { 263 return err 264 } 265 decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) 266 if err2 != nil { 267 return err2 268 } 269 decomp = append(decomp, decompData...) 270 } 271 comp := bitutil.CompressBytes(decomp) 272 273 decompSize += uint64(len(decomp)) 274 compSize += uint64(len(comp)) 275 if len(comp) > 0 { 276 b.trie.Update(encKey[:], comp) 277 } else { 278 b.trie.Delete(encKey[:]) 279 } 280 } 281 282 batch := b.cdb.NewBatch() 283 root, err := b.trie.CommitTo(batch) 284 if err != nil { 285 return err 286 } else { 287 batch.Write() 288 sectionHead := b.sectionHeads[b.bloomTrieRatio-1] 289 log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize)) 290 StoreBloomTrieRoot(b.db, b.section, sectionHead, root) 291 } 292 293 return nil 294 }