github.com/anthdm/go-ethereum@v1.8.4-0.20180412101906-60516c83b011/light/postprocess.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package light 18 19 import ( 20 "encoding/binary" 21 "errors" 22 "math/big" 23 "time" 24 25 "github.com/ethereum/go-ethereum/common" 26 "github.com/ethereum/go-ethereum/common/bitutil" 27 "github.com/ethereum/go-ethereum/core" 28 "github.com/ethereum/go-ethereum/core/types" 29 "github.com/ethereum/go-ethereum/ethdb" 30 "github.com/ethereum/go-ethereum/log" 31 "github.com/ethereum/go-ethereum/params" 32 "github.com/ethereum/go-ethereum/rlp" 33 "github.com/ethereum/go-ethereum/trie" 34 ) 35 36 const ( 37 // CHTFrequencyClient is the block frequency for creating CHTs on the client side. 38 CHTFrequencyClient = 32768 39 40 // CHTFrequencyServer is the block frequency for creating CHTs on the server side. 41 // Eventually this can be merged back with the client version, but that requires a 42 // full database upgrade, so that should be left for a suitable moment. 43 CHTFrequencyServer = 4096 44 45 HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available 46 HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated 47 ) 48 49 // trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with 50 // the appropriate section index and head hash. It is used to start light syncing from this checkpoint 51 // and avoid downloading the entire header chain while still being able to securely access old headers/logs. 52 type trustedCheckpoint struct { 53 name string 54 sectionIdx uint64 55 sectionHead, chtRoot, bloomTrieRoot common.Hash 56 } 57 58 var ( 59 mainnetCheckpoint = trustedCheckpoint{ 60 name: "mainnet", 61 sectionIdx: 161, 62 sectionHead: common.HexToHash("75b0c4baa7a62cece48abdcb03b6f31601961c9bece67dcd61df87aad4fc0d8d"), 63 chtRoot: common.HexToHash("bbbfaa67b29716348997ec21a39c03b8d1fb973f6a43740b865595ba26ee812f"), 64 bloomTrieRoot: common.HexToHash("d6db6e6248354d7453391ce97830072a28ea4216be0bd95a5db9f53b1a64677b"), 65 } 66 67 ropstenCheckpoint = trustedCheckpoint{ 68 name: "ropsten", 69 sectionIdx: 87, 70 sectionHead: common.HexToHash("ebc0adcb30ed21cbe95bd77499cc1af0bada621fee3644cb80dbcf1444c123fe"), 71 chtRoot: common.HexToHash("d9830f4893c821ddf149b8cb9d3e3bfe3109d2eea8e3c4a4ede7c8b2ee8a7800"), 72 bloomTrieRoot: common.HexToHash("c76e12d713f65b84c5a36d06bc77d0c8419248ea0b36e0812a78b76aa6da0ddb"), 73 } 74 ) 75 76 // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to 77 var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ 78 params.MainnetGenesisHash: mainnetCheckpoint, 79 params.TestnetGenesisHash: ropstenCheckpoint, 80 } 81 82 var ( 83 ErrNoTrustedCht = errors.New("No trusted canonical hash trie") 84 ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") 85 ErrNoHeader = errors.New("Header not found") 86 chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash 87 ChtTablePrefix = "cht-" 88 ) 89 90 // ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format 91 type ChtNode struct { 92 Hash common.Hash 93 Td *big.Int 94 } 95 96 // GetChtRoot reads the CHT root assoctiated to the given section from the database 97 // Note that sectionIdx is specified according to LES/1 CHT section size 98 func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 99 var encNumber [8]byte 100 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 101 data, _ := db.Get(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...)) 102 return common.BytesToHash(data) 103 } 104 105 // GetChtV2Root reads the CHT root assoctiated to the given section from the database 106 // Note that sectionIdx is specified according to LES/2 CHT section size 107 func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 108 return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) 109 } 110 111 // StoreChtRoot writes the CHT root assoctiated to the given section into the database 112 // Note that sectionIdx is specified according to LES/1 CHT section size 113 func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 114 var encNumber [8]byte 115 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 116 db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 117 } 118 119 // ChtIndexerBackend implements core.ChainIndexerBackend 120 type ChtIndexerBackend struct { 121 diskdb ethdb.Database 122 triedb *trie.Database 123 section, sectionSize uint64 124 lastHash common.Hash 125 trie *trie.Trie 126 } 127 128 // NewBloomTrieIndexer creates a BloomTrie chain indexer 129 func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 130 var sectionSize, confirmReq uint64 131 if clientMode { 132 sectionSize = CHTFrequencyClient 133 confirmReq = HelperTrieConfirmations 134 } else { 135 sectionSize = CHTFrequencyServer 136 confirmReq = HelperTrieProcessConfirmations 137 } 138 idb := ethdb.NewTable(db, "chtIndex-") 139 backend := &ChtIndexerBackend{ 140 diskdb: db, 141 triedb: trie.NewDatabase(ethdb.NewTable(db, ChtTablePrefix)), 142 sectionSize: sectionSize, 143 } 144 return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") 145 } 146 147 // Reset implements core.ChainIndexerBackend 148 func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 149 var root common.Hash 150 if section > 0 { 151 root = GetChtRoot(c.diskdb, section-1, lastSectionHead) 152 } 153 var err error 154 c.trie, err = trie.New(root, c.triedb) 155 c.section = section 156 return err 157 } 158 159 // Process implements core.ChainIndexerBackend 160 func (c *ChtIndexerBackend) Process(header *types.Header) { 161 hash, num := header.Hash(), header.Number.Uint64() 162 c.lastHash = hash 163 164 td := core.GetTd(c.diskdb, hash, num) 165 if td == nil { 166 panic(nil) 167 } 168 var encNumber [8]byte 169 binary.BigEndian.PutUint64(encNumber[:], num) 170 data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) 171 c.trie.Update(encNumber[:], data) 172 } 173 174 // Commit implements core.ChainIndexerBackend 175 func (c *ChtIndexerBackend) Commit() error { 176 root, err := c.trie.Commit(nil) 177 if err != nil { 178 return err 179 } 180 c.triedb.Commit(root, false) 181 182 if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { 183 log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) 184 } 185 StoreChtRoot(c.diskdb, c.section, c.lastHash, root) 186 return nil 187 } 188 189 const ( 190 BloomTrieFrequency = 32768 191 ethBloomBitsSection = 4096 192 ethBloomBitsConfirmations = 256 193 ) 194 195 var ( 196 bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash 197 BloomTrieTablePrefix = "blt-" 198 ) 199 200 // GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database 201 func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { 202 var encNumber [8]byte 203 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 204 data, _ := db.Get(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) 205 return common.BytesToHash(data) 206 } 207 208 // StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database 209 func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { 210 var encNumber [8]byte 211 binary.BigEndian.PutUint64(encNumber[:], sectionIdx) 212 db.Put(append(append(bloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) 213 } 214 215 // BloomTrieIndexerBackend implements core.ChainIndexerBackend 216 type BloomTrieIndexerBackend struct { 217 diskdb ethdb.Database 218 triedb *trie.Database 219 section, parentSectionSize, bloomTrieRatio uint64 220 trie *trie.Trie 221 sectionHeads []common.Hash 222 } 223 224 // NewBloomTrieIndexer creates a BloomTrie chain indexer 225 func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { 226 backend := &BloomTrieIndexerBackend{ 227 diskdb: db, 228 triedb: trie.NewDatabase(ethdb.NewTable(db, BloomTrieTablePrefix)), 229 } 230 idb := ethdb.NewTable(db, "bltIndex-") 231 232 var confirmReq uint64 233 if clientMode { 234 backend.parentSectionSize = BloomTrieFrequency 235 confirmReq = HelperTrieConfirmations 236 } else { 237 backend.parentSectionSize = ethBloomBitsSection 238 confirmReq = HelperTrieProcessConfirmations 239 } 240 backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize 241 backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) 242 return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") 243 } 244 245 // Reset implements core.ChainIndexerBackend 246 func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { 247 var root common.Hash 248 if section > 0 { 249 root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) 250 } 251 var err error 252 b.trie, err = trie.New(root, b.triedb) 253 b.section = section 254 return err 255 } 256 257 // Process implements core.ChainIndexerBackend 258 func (b *BloomTrieIndexerBackend) Process(header *types.Header) { 259 num := header.Number.Uint64() - b.section*BloomTrieFrequency 260 if (num+1)%b.parentSectionSize == 0 { 261 b.sectionHeads[num/b.parentSectionSize] = header.Hash() 262 } 263 } 264 265 // Commit implements core.ChainIndexerBackend 266 func (b *BloomTrieIndexerBackend) Commit() error { 267 var compSize, decompSize uint64 268 269 for i := uint(0); i < types.BloomBitLength; i++ { 270 var encKey [10]byte 271 binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) 272 binary.BigEndian.PutUint64(encKey[2:10], b.section) 273 var decomp []byte 274 for j := uint64(0); j < b.bloomTrieRatio; j++ { 275 data, err := core.GetBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) 276 if err != nil { 277 return err 278 } 279 decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) 280 if err2 != nil { 281 return err2 282 } 283 decomp = append(decomp, decompData...) 284 } 285 comp := bitutil.CompressBytes(decomp) 286 287 decompSize += uint64(len(decomp)) 288 compSize += uint64(len(comp)) 289 if len(comp) > 0 { 290 b.trie.Update(encKey[:], comp) 291 } else { 292 b.trie.Delete(encKey[:]) 293 } 294 } 295 root, err := b.trie.Commit(nil) 296 if err != nil { 297 return err 298 } 299 b.triedb.Commit(root, false) 300 301 sectionHead := b.sectionHeads[b.bloomTrieRatio-1] 302 log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) 303 StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) 304 305 return nil 306 }