github.com/niluplatform/go-nilu@v1.7.4-0.20200912082737-a0cb0776d52c/light/postprocess.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package light 18 19 import ( 20 "encoding/binary" 21 "errors" 22 "math/big" 23 "time" 24 25 "github.com/NiluPlatform/go-nilu/common" 26 "github.com/NiluPlatform/go-nilu/common/bitutil" 27 "github.com/NiluPlatform/go-nilu/core" 28 "github.com/NiluPlatform/go-nilu/core/rawdb" 29 "github.com/NiluPlatform/go-nilu/core/types" 30 "github.com/NiluPlatform/go-nilu/ethdb" 31 "github.com/NiluPlatform/go-nilu/log" 32 "github.com/NiluPlatform/go-nilu/params" 33 "github.com/NiluPlatform/go-nilu/rlp" 34 "github.com/NiluPlatform/go-nilu/trie" 35 ) 36 37 const ( 38 // CHTFrequencyClient is the block frequency for creating CHTs on the client side. 39 CHTFrequencyClient = 32768 40 41 // CHTFrequencyServer is the block frequency for creating CHTs on the server side. 42 // Eventually this can be merged back with the client version, but that requires a 43 // full database upgrade, so that should be left for a suitable moment. 44 CHTFrequencyServer = 4096 45 46 HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available 47 HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated 48 ) 49 50 // trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with 51 // the appropriate section index and head hash. It is used to start light syncing from this checkpoint 52 // and avoid downloading the entire header chain while still being able to securely access old headers/logs. 53 type trustedCheckpoint struct { 54 name string 55 sectionIdx uint64 56 sectionHead, chtRoot, bloomTrieRoot common.Hash 57 } 58 59 var ( 60 mainnetCheckpoint = trustedCheckpoint{ 61 name: "mainnet", 62 sectionIdx: 170, 63 sectionHead: common.HexToHash("3bb2c28bcce463d57968f14f56cdb3fbf35349ab7a701f44c1afb57349c9a356"), 64 chtRoot: common.HexToHash("d92b6d0853455f8439086292338e87f69781921680dd7aa072fb71547b87415e"), 65 bloomTrieRoot: common.HexToHash("e4e8250a2fefddead7ae42daecd848cbf9b66d748a8270f8bbd4370b764bb9e9"), 66 } 67 68 ropstenCheckpoint = trustedCheckpoint{ 69 name: "ropsten", 70 sectionIdx: 97, 71 sectionHead: common.HexToHash("719448c67c01eb5b9f27833a36a4e34612f66801316d7ff37daf9e77fb4cd095"), 72 chtRoot: common.HexToHash("a7857afc15930ca6e583b6c3d563a025144011655843d52d28e2fdaadd417bea"), 73 bloomTrieRoot: common.HexToHash("9c71d4b50cbec86dfeaa8e08992de8a4667b81d13c54d6522b17ce2fc5d36416"), 74 } 75 ) 76 77 // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to 78 var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ 79 params.MainnetGenesisHash: mainnetCheckpoint, 80 params.TestnetGenesisHash: ropstenCheckpoint, 81 } 82 83 var ( 84 ErrNoTrustedCht = errors.New("No trusted canonical hash trie") 85 ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") 86 ErrNoHeader = errors.New("Header not found") 87 chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash 88 ChtTablePrefix = "cht-" 89 ) 90 91 // ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format 92 type ChtNode struct { 93 Hash common.Hash 94 Td *big.Int 95 } 96 97 // GetChtRoot reads the CHT root assoctiated to the given section from the database 98 // Note that sectionIdx is specified according to LES/1 CHT section size 99 func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 100 var encNumber [8]byte 101 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 102 data, _ := db.Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...)) 103 return common.BytesToHash(data) 104 } 105 106 // GetChtV2Root reads the CHT root assoctiated to the given section from the database 107 // Note that sectionIdx is specified according to LES/2 CHT section size 108 func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 109 return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) 110 } 111 112 // StoreChtRoot writes the CHT root assoctiated to the given section into the database 113 // Note that sectionIdx is specified according to LES/1 CHT section size 114 func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 115 var encNumber [8]byte 116 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 117 db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 118 } 119 120 // ChtIndexerBackend implements core.ChainIndexerBackend 121 type ChtIndexerBackend struct { 122 diskdb ethdb.Database 123 triedb *trie.Database 124 section, sectionSize uint64 125 lastHash common.Hash 126 trie *trie.Trie 127 } 128 129 // NewBloomTrieIndexer creates a BloomTrie chain indexer 130 func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 131 var sectionSize, confirmReq uint64 132 if clientMode { 133 sectionSize = CHTFrequencyClient 134 confirmReq = HelperTrieConfirmations 135 } else { 136 sectionSize = CHTFrequencyServer 137 confirmReq = HelperTrieProcessConfirmations 138 } 139 idb := ethdb.NewTable(db, "chtIndex-") 140 backend := &ChtIndexerBackend{ 141 diskdb: db, 142 triedb: trie.NewDatabase(ethdb.NewTable(db, ChtTablePrefix)), 143 sectionSize: sectionSize, 144 } 145 return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") 146 } 147 148 // Reset implements core.ChainIndexerBackend 149 func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 150 var root common.Hash 151 if section > 0 { 152 root = GetChtRoot(c.diskdb, section-1, lastSectionHead) 153 } 154 var err error 155 c.trie, err = trie.New(root, c.triedb) 156 c.section = section 157 return err 158 } 159 160 // Process implements core.ChainIndexerBackend 161 func (c *ChtIndexerBackend) Process(header *types.Header) { 162 hash, num := header.Hash(), header.Number.Uint64() 163 c.lastHash = hash 164 165 td := rawdb.ReadTd(c.diskdb, hash, num) 166 if td == nil { 167 panic(nil) 168 } 169 var encNumber [8]byte 170 binary.BigEndian.PutUint64(encNumber[:], num) 171 data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) 172 c.trie.Update(encNumber[:], data) 173 } 174 175 // Commit implements core.ChainIndexerBackend 176 func (c *ChtIndexerBackend) Commit() error { 177 root, err := c.trie.Commit(nil) 178 if err != nil { 179 return err 180 } 181 c.triedb.Commit(root, false) 182 183 if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { 184 log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) 185 } 186 StoreChtRoot(c.diskdb, c.section, c.lastHash, root) 187 return nil 188 } 189 190 const ( 191 BloomTrieFrequency = 32768 192 ethBloomBitsSection = 4096 193 ethBloomBitsConfirmations = 256 194 ) 195 196 var ( 197 bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash 198 BloomTrieTablePrefix = "blt-" 199 ) 200 201 // GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database 202 func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 203 var encNumber [8]byte 204 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 205 data, _ := db.Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) 206 return common.BytesToHash(data) 207 } 208 209 // StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database 210 func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 211 var encNumber [8]byte 212 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 213 db.Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 214 } 215 216 // BloomTrieIndexerBackend implements core.ChainIndexerBackend 217 type BloomTrieIndexerBackend struct { 218 diskdb ethdb.Database 219 triedb *trie.Database 220 section, parentSectionSize, bloomTrieRatio uint64 221 trie *trie.Trie 222 sectionHeads []common.Hash 223 } 224 225 // NewBloomTrieIndexer creates a BloomTrie chain indexer 226 func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 227 backend := &BloomTrieIndexerBackend{ 228 diskdb: db, 229 triedb: trie.NewDatabase(ethdb.NewTable(db, BloomTrieTablePrefix)), 230 } 231 idb := ethdb.NewTable(db, "bltIndex-") 232 233 var confirmReq uint64 234 if clientMode { 235 backend.parentSectionSize = BloomTrieFrequency 236 confirmReq = HelperTrieConfirmations 237 } else { 238 backend.parentSectionSize = ethBloomBitsSection 239 confirmReq = HelperTrieProcessConfirmations 240 } 241 backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize 242 backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) 243 return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") 244 } 245 246 // Reset implements core.ChainIndexerBackend 247 func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 248 var root common.Hash 249 if section > 0 { 250 root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) 251 } 252 var err error 253 b.trie, err = trie.New(root, b.triedb) 254 b.section = section 255 return err 256 } 257 258 // Process implements core.ChainIndexerBackend 259 func (b *BloomTrieIndexerBackend) Process(header *types.Header) { 260 num := header.Number.Uint64() - b.section*BloomTrieFrequency 261 if (num+1)%b.parentSectionSize == 0 { 262 b.sectionHeads[num/b.parentSectionSize] = header.Hash() 263 } 264 } 265 266 // Commit implements core.ChainIndexerBackend 267 func (b *BloomTrieIndexerBackend) Commit() error { 268 var compSize, decompSize uint64 269 270 for i := uint(0); i < types.BloomBitLength; i++ { 271 var encKey [10]byte 272 binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) 273 binary.BigEndian.PutUint64(encKey[2:10], b.section) 274 var decomp []byte 275 for j := uint64(0); j < b.bloomTrieRatio; j++ { 276 data, err := rawdb.ReadBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) 277 if err != nil { 278 return err 279 } 280 decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) 281 if err2 != nil { 282 return err2 283 } 284 decomp = append(decomp, decompData...) 285 } 286 comp := bitutil.CompressBytes(decomp) 287 288 decompSize += uint64(len(decomp)) 289 compSize += uint64(len(comp)) 290 if len(comp) > 0 { 291 b.trie.Update(encKey[:], comp) 292 } else { 293 b.trie.Delete(encKey[:]) 294 } 295 } 296 root, err := b.trie.Commit(nil) 297 if err != nil { 298 return err 299 } 300 b.triedb.Commit(root, false) 301 302 sectionHead := b.sectionHeads[b.bloomTrieRatio-1] 303 log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) 304 StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) 305 306 return nil 307 }