github.com/etherbanking/go-etherbanking@v1.7.1-0.20181009210156-cf649bca5aba/core/bloombits/matcher_test.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package bloombits 18 19 import ( 20 "math/rand" 21 "sync/atomic" 22 "testing" 23 "time" 24 ) 25 26 const testSectionSize = 4096 27 28 // Tests the matcher pipeline on a single continuous workflow without interrupts. 29 func TestMatcherContinuous(t *testing.T) { 30 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75) 31 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81) 32 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36) 33 } 34 35 // Tests the matcher pipeline on a constantly interrupted and resumed work pattern 36 // with the aim of ensuring data items are requested only once. 37 func TestMatcherIntermittent(t *testing.T) { 38 testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75) 39 testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81) 40 testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36) 41 } 42 43 // Tests the matcher pipeline on random input to hopefully catch anomalies. 44 func TestMatcherRandom(t *testing.T) { 45 for i := 0; i < 10; i++ { 46 testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0) 47 testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0) 48 testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0) 49 testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0) 50 testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0) 51 } 52 } 53 54 // Tests that matching on everything doesn't crash (special case internally). 55 func TestWildcardMatcher(t *testing.T) { 56 testMatcherBothModes(t, nil, 10000, 0) 57 } 58 59 // makeRandomIndexes generates a random filter system, composed on multiple filter 60 // criteria, each having one bloom list component for the address and arbitrarilly 61 // many topic bloom list components. 62 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { 63 res := make([][]bloomIndexes, len(lengths)) 64 for i, topics := range lengths { 65 res[i] = make([]bloomIndexes, topics) 66 for j := 0; j < topics; j++ { 67 for k := 0; k < len(res[i][j]); k++ { 68 res[i][j][k] = uint(rand.Intn(max-1) + 2) 69 } 70 } 71 } 72 return res 73 } 74 75 // testMatcherDiffBatches runs the given matches test in single-delivery and also 76 // in batches delivery mode, verifying that all kinds of deliveries are handled 77 // correctly withn. 78 func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) { 79 singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1) 80 batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16) 81 82 if singleton != batched { 83 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) 84 } 85 } 86 87 // testMatcherBothModes runs the given matcher test in both continuous as well as 88 // in intermittent mode, verifying that the request counts match each other. 89 func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) { 90 continuous := testMatcher(t, filter, blocks, false, retrievals, 16) 91 intermittent := testMatcher(t, filter, blocks, true, retrievals, 16) 92 93 if continuous != intermittent { 94 t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) 95 } 96 } 97 98 // testMatcher is a generic tester to run the given matcher test and return the 99 // number of requests made for cross validation between different modes. 100 func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { 101 // Create a new matcher an simulate our explicit random bitsets 102 matcher := NewMatcher(testSectionSize, nil) 103 matcher.filters = filter 104 105 for _, rule := range filter { 106 for _, topic := range rule { 107 for _, bit := range topic { 108 matcher.addScheduler(bit) 109 } 110 } 111 } 112 // Track the number of retrieval requests made 113 var requested uint32 114 115 // Start the matching session for the filter and the retriver goroutines 116 quit := make(chan struct{}) 117 matches := make(chan uint64, 16) 118 119 session, err := matcher.Start(0, blocks-1, matches) 120 if err != nil { 121 t.Fatalf("failed to stat matcher session: %v", err) 122 } 123 startRetrievers(session, quit, &requested, maxReqCount) 124 125 // Iterate over all the blocks and verify that the pipeline produces the correct matches 126 for i := uint64(0); i < blocks; i++ { 127 if expMatch3(filter, i) { 128 match, ok := <-matches 129 if !ok { 130 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) 131 return 0 132 } 133 if match != i { 134 t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) 135 } 136 // If we're testing intermittent mode, abort and restart the pipeline 137 if intermittent { 138 session.Close(time.Second) 139 close(quit) 140 141 quit = make(chan struct{}) 142 matches = make(chan uint64, 16) 143 144 session, err = matcher.Start(i+1, blocks-1, matches) 145 if err != nil { 146 t.Fatalf("failed to stat matcher session: %v", err) 147 } 148 startRetrievers(session, quit, &requested, maxReqCount) 149 } 150 } 151 } 152 // Ensure the result channel is torn down after the last block 153 match, ok := <-matches 154 if ok { 155 t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) 156 } 157 // Clean up the session and ensure we match the expected retrieval count 158 session.Close(time.Second) 159 close(quit) 160 161 if retrievals != 0 && requested != retrievals { 162 t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals) 163 } 164 return requested 165 } 166 167 // startRetrievers starts a batch of goroutines listening for section requests 168 // and serving them. 169 func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) { 170 requests := make(chan chan *Retrieval) 171 172 for i := 0; i < 10; i++ { 173 // Start a multiplexer to test multiple threaded execution 174 go session.Multiplex(batch, 100*time.Microsecond, requests) 175 176 // Start a services to match the above multiplexer 177 go func() { 178 for { 179 // Wait for a service request or a shutdown 180 select { 181 case <-quit: 182 return 183 184 case request := <-requests: 185 task := <-request 186 187 task.Bitsets = make([][]byte, len(task.Sections)) 188 for i, section := range task.Sections { 189 if rand.Int()%4 != 0 { // Handle occasional missing deliveries 190 task.Bitsets[i] = generateBitset(task.Bit, section) 191 atomic.AddUint32(retrievals, 1) 192 } 193 } 194 request <- task 195 } 196 } 197 }() 198 } 199 } 200 201 // generateBitset generates the rotated bitset for the given bloom bit and section 202 // numbers. 203 func generateBitset(bit uint, section uint64) []byte { 204 bitset := make([]byte, testSectionSize/8) 205 for i := 0; i < len(bitset); i++ { 206 for b := 0; b < 8; b++ { 207 blockIdx := section*testSectionSize + uint64(i*8+b) 208 bitset[i] += bitset[i] 209 if (blockIdx % uint64(bit)) == 0 { 210 bitset[i]++ 211 } 212 } 213 } 214 return bitset 215 } 216 217 func expMatch1(filter bloomIndexes, i uint64) bool { 218 for _, ii := range filter { 219 if (i % uint64(ii)) != 0 { 220 return false 221 } 222 } 223 return true 224 } 225 226 func expMatch2(filter []bloomIndexes, i uint64) bool { 227 for _, ii := range filter { 228 if expMatch1(ii, i) { 229 return true 230 } 231 } 232 return false 233 } 234 235 func expMatch3(filter [][]bloomIndexes, i uint64) bool { 236 for _, ii := range filter { 237 if !expMatch2(ii, i) { 238 return false 239 } 240 } 241 return true 242 }