github.com/n1ghtfa1l/go-vnt@v0.6.4-alpha.6/core/bloombits/matcher_test.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package bloombits 18 19 import ( 20 "context" 21 "math/rand" 22 "sync/atomic" 23 "testing" 24 "time" 25 26 "github.com/vntchain/go-vnt/common" 27 ) 28 29 const testSectionSize = 4096 30 31 // Tests that wildcard filter rules (nil) can be specified and are handled well. 32 func TestMatcherWildcards(t *testing.T) { 33 matcher := NewMatcher(testSectionSize, [][][]byte{ 34 {common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard 35 {common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard 36 {common.Hash{0x01}.Bytes()}, // Plain rule, sanity check 37 {common.Hash{0x01}.Bytes(), nil}, // Wildcard suffix, drop rule 38 {nil, common.Hash{0x01}.Bytes()}, // Wildcard prefix, drop rule 39 {nil, nil}, // Wildcard combo, drop rule 40 {}, // Inited wildcard rule, drop rule 41 nil, // Proper wildcard rule, drop rule 42 }) 43 if len(matcher.filters) != 3 { 44 t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3) 45 } 46 if len(matcher.filters[0]) != 2 { 47 t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2) 48 } 49 if len(matcher.filters[1]) != 2 { 50 t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2) 51 } 52 if len(matcher.filters[2]) != 1 { 53 t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1) 54 } 55 } 56 57 // Tests the matcher pipeline on a single continuous workflow without interrupts. 58 func TestMatcherContinuous(t *testing.T) { 59 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75) 60 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81) 61 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36) 62 } 63 64 // Tests the matcher pipeline on a constantly interrupted and resumed work pattern 65 // with the aim of ensuring data items are requested only once. 66 func TestMatcherIntermittent(t *testing.T) { 67 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75) 68 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81) 69 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36) 70 } 71 72 // Tests the matcher pipeline on random input to hopefully catch anomalies. 73 func TestMatcherRandom(t *testing.T) { 74 for i := 0; i < 10; i++ { 75 testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0) 76 testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0) 77 testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0) 78 testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0) 79 testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0) 80 } 81 } 82 83 // Tests that the matcher can properly find matches if the starting block is 84 // shifter from a multiple of 8. 85 func TestMatcherShifted(t *testing.T) { 86 // Block 0 always matches in the tests, skip ahead of first 8 blocks with the 87 // start to get a potential zero byte in the matcher bitset. 88 89 // To keep the second bitset byte zero, the filter must only match for the first 90 // time in block 16, so doing an all-16 bit filter should suffice. 91 92 // To keep the starting block non divisible by 8, block number 9 is the first 93 // that would introduce a shift and not match block 0. 94 testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0) 95 } 96 97 // Tests that matching on everything doesn't crash (special case internally). 98 func TestWildcardMatcher(t *testing.T) { 99 testMatcherBothModes(t, nil, 0, 10000, 0) 100 } 101 102 // makeRandomIndexes generates a random filter system, composed on multiple filter 103 // criteria, each having one bloom list component for the address and arbitrarily 104 // many topic bloom list components. 105 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { 106 res := make([][]bloomIndexes, len(lengths)) 107 for i, topics := range lengths { 108 res[i] = make([]bloomIndexes, topics) 109 for j := 0; j < topics; j++ { 110 for k := 0; k < len(res[i][j]); k++ { 111 res[i][j][k] = uint(rand.Intn(max-1) + 2) 112 } 113 } 114 } 115 return res 116 } 117 118 // testMatcherDiffBatches runs the given matches test in single-delivery and also 119 // in batches delivery mode, verifying that all kinds of deliveries are handled 120 // correctly withn. 121 func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { 122 singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) 123 batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) 124 125 if singleton != batched { 126 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) 127 } 128 } 129 130 // testMatcherBothModes runs the given matcher test in both continuous as well as 131 // in intermittent mode, verifying that the request counts match each other. 132 func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) { 133 continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16) 134 intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16) 135 136 if continuous != intermittent { 137 t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) 138 } 139 } 140 141 // testMatcher is a generic tester to run the given matcher test and return the 142 // number of requests made for cross validation between different modes. 143 func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { 144 // Create a new matcher an simulate our explicit random bitsets 145 matcher := NewMatcher(testSectionSize, nil) 146 matcher.filters = filter 147 148 for _, rule := range filter { 149 for _, topic := range rule { 150 for _, bit := range topic { 151 matcher.addScheduler(bit) 152 } 153 } 154 } 155 // Track the number of retrieval requests made 156 var requested uint32 157 158 // Start the matching session for the filter and the retriver goroutines 159 quit := make(chan struct{}) 160 matches := make(chan uint64, 16) 161 162 session, err := matcher.Start(context.Background(), start, blocks-1, matches) 163 if err != nil { 164 t.Fatalf("failed to stat matcher session: %v", err) 165 } 166 startRetrievers(session, quit, &requested, maxReqCount) 167 168 // Iterate over all the blocks and verify that the pipeline produces the correct matches 169 for i := start; i < blocks; i++ { 170 if expMatch3(filter, i) { 171 match, ok := <-matches 172 if !ok { 173 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) 174 return 0 175 } 176 if match != i { 177 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) 178 } 179 // If we're testing intermittent mode, abort and restart the pipeline 180 if intermittent { 181 session.Close() 182 close(quit) 183 184 quit = make(chan struct{}) 185 matches = make(chan uint64, 16) 186 187 session, err = matcher.Start(context.Background(), i+1, blocks-1, matches) 188 if err != nil { 189 t.Fatalf("failed to stat matcher session: %v", err) 190 } 191 startRetrievers(session, quit, &requested, maxReqCount) 192 } 193 } 194 } 195 // Ensure the result channel is torn down after the last block 196 match, ok := <-matches 197 if ok { 198 t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) 199 } 200 // Clean up the session and ensure we match the expected retrieval count 201 session.Close() 202 close(quit) 203 204 if retrievals != 0 && requested != retrievals { 205 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals) 206 } 207 return requested 208 } 209 210 // startRetrievers starts a batch of goroutines listening for section requests 211 // and serving them. 212 func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) { 213 requests := make(chan chan *Retrieval) 214 215 for i := 0; i < 10; i++ { 216 // Start a multiplexer to test multiple threaded execution 217 go session.Multiplex(batch, 100*time.Microsecond, requests) 218 219 // Start a services to match the above multiplexer 220 go func() { 221 for { 222 // Wait for a service request or a shutdown 223 select { 224 case <-quit: 225 return 226 227 case request := <-requests: 228 task := <-request 229 230 task.Bitsets = make([][]byte, len(task.Sections)) 231 for i, section := range task.Sections { 232 if rand.Int()%4 != 0 { // Handle occasional missing deliveries 233 task.Bitsets[i] = generateBitset(task.Bit, section) 234 atomic.AddUint32(retrievals, 1) 235 } 236 } 237 request <- task 238 } 239 } 240 }() 241 } 242 } 243 244 // generateBitset generates the rotated bitset for the given bloom bit and section 245 // numbers. 246 func generateBitset(bit uint, section uint64) []byte { 247 bitset := make([]byte, testSectionSize/8) 248 for i := 0; i < len(bitset); i++ { 249 for b := 0; b < 8; b++ { 250 blockIdx := section*testSectionSize + uint64(i*8+b) 251 bitset[i] += bitset[i] 252 if (blockIdx % uint64(bit)) == 0 { 253 bitset[i]++ 254 } 255 } 256 } 257 return bitset 258 } 259 260 func expMatch1(filter bloomIndexes, i uint64) bool { 261 for _, ii := range filter { 262 if (i % uint64(ii)) != 0 { 263 return false 264 } 265 } 266 return true 267 } 268 269 func expMatch2(filter []bloomIndexes, i uint64) bool { 270 for _, ii := range filter { 271 if expMatch1(ii, i) { 272 return true 273 } 274 } 275 return false 276 } 277 278 func expMatch3(filter [][]bloomIndexes, i uint64) bool { 279 for _, ii := range filter { 280 if !expMatch2(ii, i) { 281 return false 282 } 283 } 284 return true 285 }