github.com/ImPedro29/bor@v0.2.7/trie/sync_bloom.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package trie 18 19 import ( 20 "encoding/binary" 21 "fmt" 22 "sync" 23 "sync/atomic" 24 "time" 25 26 "github.com/ethereum/go-ethereum/common" 27 "github.com/ethereum/go-ethereum/core/rawdb" 28 "github.com/ethereum/go-ethereum/ethdb" 29 "github.com/ethereum/go-ethereum/log" 30 "github.com/ethereum/go-ethereum/metrics" 31 bloomfilter "github.com/holiman/bloomfilter/v2" 32 ) 33 34 var ( 35 bloomAddMeter = metrics.NewRegisteredMeter("trie/bloom/add", nil) 36 bloomLoadMeter = metrics.NewRegisteredMeter("trie/bloom/load", nil) 37 bloomTestMeter = metrics.NewRegisteredMeter("trie/bloom/test", nil) 38 bloomMissMeter = metrics.NewRegisteredMeter("trie/bloom/miss", nil) 39 bloomFaultMeter = metrics.NewRegisteredMeter("trie/bloom/fault", nil) 40 bloomErrorGauge = metrics.NewRegisteredGauge("trie/bloom/error", nil) 41 ) 42 43 // SyncBloom is a bloom filter used during fast sync to quickly decide if a trie 44 // node or contract code already exists on disk or not. It self populates from the 45 // provided disk database on creation in a background thread and will only start 46 // returning live results once that's finished. 47 type SyncBloom struct { 48 bloom *bloomfilter.Filter 49 inited uint32 50 closer sync.Once 51 closed uint32 52 pend sync.WaitGroup 53 } 54 55 // NewSyncBloom creates a new bloom filter of the given size (in megabytes) and 56 // initializes it from the database. The bloom is hard coded to use 3 filters. 57 func NewSyncBloom(memory uint64, database ethdb.Iteratee) *SyncBloom { 58 // Create the bloom filter to track known trie nodes 59 bloom, err := bloomfilter.New(memory*1024*1024*8, 4) 60 if err != nil { 61 panic(fmt.Sprintf("failed to create bloom: %v", err)) 62 } 63 log.Info("Allocated fast sync bloom", "size", common.StorageSize(memory*1024*1024)) 64 65 // Assemble the fast sync bloom and init it from previous sessions 66 b := &SyncBloom{ 67 bloom: bloom, 68 } 69 b.pend.Add(2) 70 go func() { 71 defer b.pend.Done() 72 b.init(database) 73 }() 74 go func() { 75 defer b.pend.Done() 76 b.meter() 77 }() 78 return b 79 } 80 81 // init iterates over the database, pushing every trie hash into the bloom filter. 82 func (b *SyncBloom) init(database ethdb.Iteratee) { 83 // Iterate over the database, but restart every now and again to avoid holding 84 // a persistent snapshot since fast sync can push a ton of data concurrently, 85 // bloating the disk. 86 // 87 // Note, this is fine, because everything inserted into leveldb by fast sync is 88 // also pushed into the bloom directly, so we're not missing anything when the 89 // iterator is swapped out for a new one. 90 it := database.NewIterator(nil, nil) 91 92 var ( 93 start = time.Now() 94 swap = time.Now() 95 ) 96 for it.Next() && atomic.LoadUint32(&b.closed) == 0 { 97 // If the database entry is a trie node, add it to the bloom 98 key := it.Key() 99 if len(key) == common.HashLength { 100 b.bloom.AddHash(binary.BigEndian.Uint64(key)) 101 bloomLoadMeter.Mark(1) 102 } else if ok, hash := rawdb.IsCodeKey(key); ok { 103 // If the database entry is a contract code, add it to the bloom 104 b.bloom.AddHash(binary.BigEndian.Uint64(hash)) 105 bloomLoadMeter.Mark(1) 106 } 107 // If enough time elapsed since the last iterator swap, restart 108 if time.Since(swap) > 8*time.Second { 109 key := common.CopyBytes(it.Key()) 110 111 it.Release() 112 it = database.NewIterator(nil, key) 113 114 log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start))) 115 swap = time.Now() 116 } 117 } 118 it.Release() 119 120 // Mark the bloom filter inited and return 121 log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start))) 122 atomic.StoreUint32(&b.inited, 1) 123 } 124 125 // meter periodically recalculates the false positive error rate of the bloom 126 // filter and reports it in a metric. 127 func (b *SyncBloom) meter() { 128 for { 129 // Report the current error ration. No floats, lame, scale it up. 130 bloomErrorGauge.Update(int64(b.bloom.FalsePosititveProbability() * 100000)) 131 132 // Wait one second, but check termination more frequently 133 for i := 0; i < 10; i++ { 134 if atomic.LoadUint32(&b.closed) == 1 { 135 return 136 } 137 time.Sleep(100 * time.Millisecond) 138 } 139 } 140 } 141 142 // Close terminates any background initializer still running and releases all the 143 // memory allocated for the bloom. 144 func (b *SyncBloom) Close() error { 145 b.closer.Do(func() { 146 // Ensure the initializer is stopped 147 atomic.StoreUint32(&b.closed, 1) 148 b.pend.Wait() 149 150 // Wipe the bloom, but mark it "uninited" just in case someone attempts an access 151 log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability()) 152 153 atomic.StoreUint32(&b.inited, 0) 154 b.bloom = nil 155 }) 156 return nil 157 } 158 159 // Add inserts a new trie node hash into the bloom filter. 160 func (b *SyncBloom) Add(hash []byte) { 161 if atomic.LoadUint32(&b.closed) == 1 { 162 return 163 } 164 b.bloom.AddHash(binary.BigEndian.Uint64(hash)) 165 bloomAddMeter.Mark(1) 166 } 167 168 // Contains tests if the bloom filter contains the given hash: 169 // - false: the bloom definitely does not contain hash 170 // - true: the bloom maybe contains hash 171 // 172 // While the bloom is being initialized, any query will return true. 173 func (b *SyncBloom) Contains(hash []byte) bool { 174 bloomTestMeter.Mark(1) 175 if atomic.LoadUint32(&b.inited) == 0 { 176 // We didn't load all the trie nodes from the previous run of Geth yet. As 177 // such, we can't say for sure if a hash is not present for anything. Until 178 // the init is done, we're faking "possible presence" for everything. 179 return true 180 } 181 // Bloom initialized, check the real one and report any successful misses 182 maybe := b.bloom.ContainsHash(binary.BigEndian.Uint64(hash)) 183 if !maybe { 184 bloomMissMeter.Mark(1) 185 } 186 return maybe 187 }