github.com/4000d/go-ethereum@v1.8.2-0.20180223170251-423c8bb1d821/core/bloombits/matcher_test.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package bloombits 18 19 import ( 20 "context" 21 "math/rand" 22 "sync/atomic" 23 "testing" 24 "time" 25 26 "github.com/ethereum/go-ethereum/common" 27 ) 28 29 const testSectionSize = 4096 30 31 // Tests that wildcard filter rules (nil) can be specified and are handled well. 32 func TestMatcherWildcards(t *testing.T) { 33 matcher := NewMatcher(testSectionSize, [][][]byte{ 34 {common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard 35 {common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard 36 {common.Hash{0x01}.Bytes()}, // Plain rule, sanity check 37 {common.Hash{0x01}.Bytes(), nil}, // Wildcard suffix, drop rule 38 {nil, common.Hash{0x01}.Bytes()}, // Wildcard prefix, drop rule 39 {nil, nil}, // Wildcard combo, drop rule 40 {}, // Inited wildcard rule, drop rule 41 nil, // Proper wildcard rule, drop rule 42 }) 43 if len(matcher.filters) != 3 { 44 t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3) 45 } 46 if len(matcher.filters[0]) != 2 { 47 t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2) 48 } 49 if len(matcher.filters[1]) != 2 { 50 t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2) 51 } 52 if len(matcher.filters[2]) != 1 { 53 t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1) 54 } 55 } 56 57 // Tests the matcher pipeline on a single continuous workflow without interrupts. 58 func TestMatcherContinuous(t *testing.T) { 59 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75) 60 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81) 61 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36) 62 } 63 64 // Tests the matcher pipeline on a constantly interrupted and resumed work pattern 65 // with the aim of ensuring data items are requested only once. 66 func TestMatcherIntermittent(t *testing.T) { 67 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75) 68 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81) 69 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36) 70 } 71 72 // Tests the matcher pipeline on random input to hopefully catch anomalies. 73 func TestMatcherRandom(t *testing.T) { 74 for i := 0; i < 10; i++ { 75 testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0) 76 testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0) 77 testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0) 78 testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0) 79 testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0) 80 } 81 } 82 83 // Tests that the matcher can properly find matches if the starting block is 84 // shifter from a multiple of 8. This is needed to cover an optimisation with 85 // bitset matching https://github.com/ethereum/go-ethereum/issues/15309. 86 func TestMatcherShifted(t *testing.T) { 87 // Block 0 always matches in the tests, skip ahead of first 8 blocks with the 88 // start to get a potential zero byte in the matcher bitset. 89 90 // To keep the second bitset byte zero, the filter must only match for the first 91 // time in block 16, so doing an all-16 bit filter should suffice. 92 93 // To keep the starting block non divisible by 8, block number 9 is the first 94 // that would introduce a shift and not match block 0. 95 testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0) 96 } 97 98 // Tests that matching on everything doesn't crash (special case internally). 99 func TestWildcardMatcher(t *testing.T) { 100 testMatcherBothModes(t, nil, 0, 10000, 0) 101 } 102 103 // makeRandomIndexes generates a random filter system, composed on multiple filter 104 // criteria, each having one bloom list component for the address and arbitrarily 105 // many topic bloom list components. 106 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { 107 res := make([][]bloomIndexes, len(lengths)) 108 for i, topics := range lengths { 109 res[i] = make([]bloomIndexes, topics) 110 for j := 0; j < topics; j++ { 111 for k := 0; k < len(res[i][j]); k++ { 112 res[i][j][k] = uint(rand.Intn(max-1) + 2) 113 } 114 } 115 } 116 return res 117 } 118 119 // testMatcherDiffBatches runs the given matches test in single-delivery and also 120 // in batches delivery mode, verifying that all kinds of deliveries are handled 121 // correctly withn. 122 func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { 123 singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) 124 batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) 125 126 if singleton != batched { 127 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) 128 } 129 } 130 131 // testMatcherBothModes runs the given matcher test in both continuous as well as 132 // in intermittent mode, verifying that the request counts match each other. 133 func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) { 134 continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16) 135 intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16) 136 137 if continuous != intermittent { 138 t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) 139 } 140 } 141 142 // testMatcher is a generic tester to run the given matcher test and return the 143 // number of requests made for cross validation between different modes. 144 func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { 145 // Create a new matcher an simulate our explicit random bitsets 146 matcher := NewMatcher(testSectionSize, nil) 147 matcher.filters = filter 148 149 for _, rule := range filter { 150 for _, topic := range rule { 151 for _, bit := range topic { 152 matcher.addScheduler(bit) 153 } 154 } 155 } 156 // Track the number of retrieval requests made 157 var requested uint32 158 159 // Start the matching session for the filter and the retriver goroutines 160 quit := make(chan struct{}) 161 matches := make(chan uint64, 16) 162 163 session, err := matcher.Start(context.Background(), start, blocks-1, matches) 164 if err != nil { 165 t.Fatalf("failed to stat matcher session: %v", err) 166 } 167 startRetrievers(session, quit, &requested, maxReqCount) 168 169 // Iterate over all the blocks and verify that the pipeline produces the correct matches 170 for i := start; i < blocks; i++ { 171 if expMatch3(filter, i) { 172 match, ok := <-matches 173 if !ok { 174 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) 175 return 0 176 } 177 if match != i { 178 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) 179 } 180 // If we're testing intermittent mode, abort and restart the pipeline 181 if intermittent { 182 session.Close() 183 close(quit) 184 185 quit = make(chan struct{}) 186 matches = make(chan uint64, 16) 187 188 session, err = matcher.Start(context.Background(), i+1, blocks-1, matches) 189 if err != nil { 190 t.Fatalf("failed to stat matcher session: %v", err) 191 } 192 startRetrievers(session, quit, &requested, maxReqCount) 193 } 194 } 195 } 196 // Ensure the result channel is torn down after the last block 197 match, ok := <-matches 198 if ok { 199 t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) 200 } 201 // Clean up the session and ensure we match the expected retrieval count 202 session.Close() 203 close(quit) 204 205 if retrievals != 0 && requested != retrievals { 206 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals) 207 } 208 return requested 209 } 210 211 // startRetrievers starts a batch of goroutines listening for section requests 212 // and serving them. 213 func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) { 214 requests := make(chan chan *Retrieval) 215 216 for i := 0; i < 10; i++ { 217 // Start a multiplexer to test multiple threaded execution 218 go session.Multiplex(batch, 100*time.Microsecond, requests) 219 220 // Start a services to match the above multiplexer 221 go func() { 222 for { 223 // Wait for a service request or a shutdown 224 select { 225 case <-quit: 226 return 227 228 case request := <-requests: 229 task := <-request 230 231 task.Bitsets = make([][]byte, len(task.Sections)) 232 for i, section := range task.Sections { 233 if rand.Int()%4 != 0 { // Handle occasional missing deliveries 234 task.Bitsets[i] = generateBitset(task.Bit, section) 235 atomic.AddUint32(retrievals, 1) 236 } 237 } 238 request <- task 239 } 240 } 241 }() 242 } 243 } 244 245 // generateBitset generates the rotated bitset for the given bloom bit and section 246 // numbers. 247 func generateBitset(bit uint, section uint64) []byte { 248 bitset := make([]byte, testSectionSize/8) 249 for i := 0; i < len(bitset); i++ { 250 for b := 0; b < 8; b++ { 251 blockIdx := section*testSectionSize + uint64(i*8+b) 252 bitset[i] += bitset[i] 253 if (blockIdx % uint64(bit)) == 0 { 254 bitset[i]++ 255 } 256 } 257 } 258 return bitset 259 } 260 261 func expMatch1(filter bloomIndexes, i uint64) bool { 262 for _, ii := range filter { 263 if (i % uint64(ii)) != 0 { 264 return false 265 } 266 } 267 return true 268 } 269 270 func expMatch2(filter []bloomIndexes, i uint64) bool { 271 for _, ii := range filter { 272 if expMatch1(ii, i) { 273 return true 274 } 275 } 276 return false 277 } 278 279 func expMatch3(filter [][]bloomIndexes, i uint64) bool { 280 for _, ii := range filter { 281 if !expMatch2(ii, i) { 282 return false 283 } 284 } 285 return true 286 }