github.com/n1ghtfa1l/go-vnt@v0.6.4-alpha.6/light/postprocess.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package light 18 19 import ( 20 "encoding/binary" 21 "errors" 22 "math/big" 23 "time" 24 25 "github.com/vntchain/go-vnt/common" 26 "github.com/vntchain/go-vnt/common/bitutil" 27 "github.com/vntchain/go-vnt/core" 28 "github.com/vntchain/go-vnt/core/rawdb" 29 "github.com/vntchain/go-vnt/core/types" 30 "github.com/vntchain/go-vnt/log" 31 "github.com/vntchain/go-vnt/params" 32 "github.com/vntchain/go-vnt/rlp" 33 "github.com/vntchain/go-vnt/trie" 34 "github.com/vntchain/go-vnt/vntdb" 35 ) 36 37 const ( 38 // CHTFrequencyClient is the block frequency for creating CHTs on the client side. 39 CHTFrequencyClient = 32768 40 41 // CHTFrequencyServer is the block frequency for creating CHTs on the server side. 42 // Eventually this can be merged back with the client version, but that requires a 43 // full database upgrade, so that should be left for a suitable moment. 44 CHTFrequencyServer = 4096 45 46 HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available 47 HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated 48 ) 49 50 // trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with 51 // the appropriate section index and head hash. It is used to start light syncing from this checkpoint 52 // and avoid downloading the entire header chain while still being able to securely access old headers/logs. 53 type trustedCheckpoint struct { 54 name string 55 sectionIdx uint64 56 sectionHead, chtRoot, bloomTrieRoot common.Hash 57 } 58 59 var ( 60 mainnetCheckpoint = trustedCheckpoint{ 61 name: "mainnet", 62 sectionIdx: 174, 63 sectionHead: common.HexToHash("a3ef48cd8f1c3a08419f0237fc7763491fe89497b3144b17adf87c1c43664613"), 64 chtRoot: common.HexToHash("dcbeed9f4dea1b3cb75601bb27c51b9960c28e5850275402ac49a150a667296e"), 65 bloomTrieRoot: common.HexToHash("6b7497a4a03e33870a2383cb6f5e70570f12b1bf5699063baf8c71d02ca90b02"), 66 } 67 ) 68 69 // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to 70 var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ 71 params.MainnetGenesisHash: mainnetCheckpoint, 72 } 73 74 var ( 75 ErrNoTrustedCht = errors.New("No trusted canonical hash trie") 76 ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") 77 ErrNoHeader = errors.New("Header not found") 78 chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash 79 ChtTablePrefix = "cht-" 80 ) 81 82 // ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format 83 type ChtNode struct { 84 Hash common.Hash 85 Td *big.Int 86 } 87 88 // GetChtRoot reads the CHT root assoctiated to the given section from the database 89 // Note that sectionIdx is specified according to LES/1 CHT section size 90 func GetChtRoot(db vntdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 91 var encNumber [8]byte 92 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 93 data, _ := db.Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...)) 94 return common.BytesToHash(data) 95 } 96 97 // GetChtV2Root reads the CHT root assoctiated to the given section from the database 98 // Note that sectionIdx is specified according to LES/2 CHT section size 99 func GetChtV2Root(db vntdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 100 return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) 101 } 102 103 // StoreChtRoot writes the CHT root assoctiated to the given section into the database 104 // Note that sectionIdx is specified according to LES/1 CHT section size 105 func StoreChtRoot(db vntdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 106 var encNumber [8]byte 107 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 108 db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 109 } 110 111 // ChtIndexerBackend implements core.ChainIndexerBackend 112 type ChtIndexerBackend struct { 113 diskdb vntdb.Database 114 triedb *trie.Database 115 section, sectionSize uint64 116 lastHash common.Hash 117 trie *trie.Trie 118 } 119 120 // NewBloomTrieIndexer creates a BloomTrie chain indexer 121 func NewChtIndexer(db vntdb.Database, clientMode bool) *core.ChainIndexer { 122 var sectionSize, confirmReq uint64 123 if clientMode { 124 sectionSize = CHTFrequencyClient 125 confirmReq = HelperTrieConfirmations 126 } else { 127 sectionSize = CHTFrequencyServer 128 confirmReq = HelperTrieProcessConfirmations 129 } 130 idb := vntdb.NewTable(db, "chtIndex-") 131 backend := &ChtIndexerBackend{ 132 diskdb: db, 133 triedb: trie.NewDatabase(vntdb.NewTable(db, ChtTablePrefix)), 134 sectionSize: sectionSize, 135 } 136 return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") 137 } 138 139 // Reset implements core.ChainIndexerBackend 140 func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 141 var root common.Hash 142 if section > 0 { 143 root = GetChtRoot(c.diskdb, section-1, lastSectionHead) 144 } 145 var err error 146 c.trie, err = trie.New(root, c.triedb) 147 c.section = section 148 return err 149 } 150 151 // Process implements core.ChainIndexerBackend 152 func (c *ChtIndexerBackend) Process(header *types.Header) { 153 hash, num := header.Hash(), header.Number.Uint64() 154 c.lastHash = hash 155 156 td := rawdb.ReadTd(c.diskdb, hash, num) 157 if td == nil { 158 panic(nil) 159 } 160 var encNumber [8]byte 161 binary.BigEndian.PutUint64(encNumber[:], num) 162 data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) 163 c.trie.Update(encNumber[:], data) 164 } 165 166 // Commit implements core.ChainIndexerBackend 167 func (c *ChtIndexerBackend) Commit() error { 168 root, err := c.trie.Commit(nil) 169 if err != nil { 170 return err 171 } 172 c.triedb.Commit(root, false) 173 174 if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { 175 log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) 176 } 177 StoreChtRoot(c.diskdb, c.section, c.lastHash, root) 178 return nil 179 } 180 181 const ( 182 BloomTrieFrequency = 32768 183 vntBloomBitsSection = 4096 184 vntBloomBitsConfirmations = 256 185 ) 186 187 var ( 188 bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash 189 BloomTrieTablePrefix = "blt-" 190 ) 191 192 // GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database 193 func GetBloomTrieRoot(db vntdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 194 var encNumber [8]byte 195 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 196 data, _ := db.Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) 197 return common.BytesToHash(data) 198 } 199 200 // StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database 201 func StoreBloomTrieRoot(db vntdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 202 var encNumber [8]byte 203 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 204 db.Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 205 } 206 207 // BloomTrieIndexerBackend implements core.ChainIndexerBackend 208 type BloomTrieIndexerBackend struct { 209 diskdb vntdb.Database 210 triedb *trie.Database 211 section, parentSectionSize, bloomTrieRatio uint64 212 trie *trie.Trie 213 sectionHeads []common.Hash 214 } 215 216 // NewBloomTrieIndexer creates a BloomTrie chain indexer 217 func NewBloomTrieIndexer(db vntdb.Database, clientMode bool) *core.ChainIndexer { 218 backend := &BloomTrieIndexerBackend{ 219 diskdb: db, 220 triedb: trie.NewDatabase(vntdb.NewTable(db, BloomTrieTablePrefix)), 221 } 222 idb := vntdb.NewTable(db, "bltIndex-") 223 224 var confirmReq uint64 225 if clientMode { 226 backend.parentSectionSize = BloomTrieFrequency 227 confirmReq = HelperTrieConfirmations 228 } else { 229 backend.parentSectionSize = vntBloomBitsSection 230 confirmReq = HelperTrieProcessConfirmations 231 } 232 backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize 233 backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) 234 return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-vntBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") 235 } 236 237 // Reset implements core.ChainIndexerBackend 238 func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 239 var root common.Hash 240 if section > 0 { 241 root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) 242 } 243 var err error 244 b.trie, err = trie.New(root, b.triedb) 245 b.section = section 246 return err 247 } 248 249 // Process implements core.ChainIndexerBackend 250 func (b *BloomTrieIndexerBackend) Process(header *types.Header) { 251 num := header.Number.Uint64() - b.section*BloomTrieFrequency 252 if (num+1)%b.parentSectionSize == 0 { 253 b.sectionHeads[num/b.parentSectionSize] = header.Hash() 254 } 255 } 256 257 // Commit implements core.ChainIndexerBackend 258 func (b *BloomTrieIndexerBackend) Commit() error { 259 var compSize, decompSize uint64 260 261 for i := uint(0); i < types.BloomBitLength; i++ { 262 var encKey [10]byte 263 binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) 264 binary.BigEndian.PutUint64(encKey[2:10], b.section) 265 var decomp []byte 266 for j := uint64(0); j < b.bloomTrieRatio; j++ { 267 data, err := rawdb.ReadBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) 268 if err != nil { 269 return err 270 } 271 decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) 272 if err2 != nil { 273 return err2 274 } 275 decomp = append(decomp, decompData...) 276 } 277 comp := bitutil.CompressBytes(decomp) 278 279 decompSize += uint64(len(decomp)) 280 compSize += uint64(len(comp)) 281 if len(comp) > 0 { 282 b.trie.Update(encKey[:], comp) 283 } else { 284 b.trie.Delete(encKey[:]) 285 } 286 } 287 root, err := b.trie.Commit(nil) 288 if err != nil { 289 return err 290 } 291 b.triedb.Commit(root, false) 292 293 sectionHead := b.sectionHeads[b.bloomTrieRatio-1] 294 log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) 295 StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) 296 297 return nil 298 }