github.com/etherbanking/go-etherbanking@v1.7.1-0.20181009210156-cf649bca5aba/core/bloombits/matcher_test.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package bloombits
    18  
    19  import (
    20  	"math/rand"
    21  	"sync/atomic"
    22  	"testing"
    23  	"time"
    24  )
    25  
    26  const testSectionSize = 4096
    27  
    28  // Tests the matcher pipeline on a single continuous workflow without interrupts.
    29  func TestMatcherContinuous(t *testing.T) {
    30  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75)
    31  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81)
    32  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36)
    33  }
    34  
    35  // Tests the matcher pipeline on a constantly interrupted and resumed work pattern
    36  // with the aim of ensuring data items are requested only once.
    37  func TestMatcherIntermittent(t *testing.T) {
    38  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75)
    39  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81)
    40  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36)
    41  }
    42  
    43  // Tests the matcher pipeline on random input to hopefully catch anomalies.
    44  func TestMatcherRandom(t *testing.T) {
    45  	for i := 0; i < 10; i++ {
    46  		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0)
    47  		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0)
    48  		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0)
    49  		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0)
    50  		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0)
    51  	}
    52  }
    53  
    54  // Tests that matching on everything doesn't crash (special case internally).
    55  func TestWildcardMatcher(t *testing.T) {
    56  	testMatcherBothModes(t, nil, 10000, 0)
    57  }
    58  
    59  // makeRandomIndexes generates a random filter system, composed on multiple filter
    60  // criteria, each having one bloom list component for the address and arbitrarilly
    61  // many topic bloom list components.
    62  func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
    63  	res := make([][]bloomIndexes, len(lengths))
    64  	for i, topics := range lengths {
    65  		res[i] = make([]bloomIndexes, topics)
    66  		for j := 0; j < topics; j++ {
    67  			for k := 0; k < len(res[i][j]); k++ {
    68  				res[i][j][k] = uint(rand.Intn(max-1) + 2)
    69  			}
    70  		}
    71  	}
    72  	return res
    73  }
    74  
    75  // testMatcherDiffBatches runs the given matches test in single-delivery and also
    76  // in batches delivery mode, verifying that all kinds of deliveries are handled
    77  // correctly withn.
    78  func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) {
    79  	singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1)
    80  	batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16)
    81  
    82  	if singleton != batched {
    83  		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
    84  	}
    85  }
    86  
    87  // testMatcherBothModes runs the given matcher test in both continuous as well as
    88  // in intermittent mode, verifying that the request counts match each other.
    89  func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) {
    90  	continuous := testMatcher(t, filter, blocks, false, retrievals, 16)
    91  	intermittent := testMatcher(t, filter, blocks, true, retrievals, 16)
    92  
    93  	if continuous != intermittent {
    94  		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
    95  	}
    96  }
    97  
    98  // testMatcher is a generic tester to run the given matcher test and return the
    99  // number of requests made for cross validation between different modes.
   100  func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
   101  	// Create a new matcher an simulate our explicit random bitsets
   102  	matcher := NewMatcher(testSectionSize, nil)
   103  	matcher.filters = filter
   104  
   105  	for _, rule := range filter {
   106  		for _, topic := range rule {
   107  			for _, bit := range topic {
   108  				matcher.addScheduler(bit)
   109  			}
   110  		}
   111  	}
   112  	// Track the number of retrieval requests made
   113  	var requested uint32
   114  
   115  	// Start the matching session for the filter and the retriver goroutines
   116  	quit := make(chan struct{})
   117  	matches := make(chan uint64, 16)
   118  
   119  	session, err := matcher.Start(0, blocks-1, matches)
   120  	if err != nil {
   121  		t.Fatalf("failed to stat matcher session: %v", err)
   122  	}
   123  	startRetrievers(session, quit, &requested, maxReqCount)
   124  
   125  	// Iterate over all the blocks and verify that the pipeline produces the correct matches
   126  	for i := uint64(0); i < blocks; i++ {
   127  		if expMatch3(filter, i) {
   128  			match, ok := <-matches
   129  			if !ok {
   130  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
   131  				return 0
   132  			}
   133  			if match != i {
   134  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
   135  			}
   136  			// If we're testing intermittent mode, abort and restart the pipeline
   137  			if intermittent {
   138  				session.Close(time.Second)
   139  				close(quit)
   140  
   141  				quit = make(chan struct{})
   142  				matches = make(chan uint64, 16)
   143  
   144  				session, err = matcher.Start(i+1, blocks-1, matches)
   145  				if err != nil {
   146  					t.Fatalf("failed to stat matcher session: %v", err)
   147  				}
   148  				startRetrievers(session, quit, &requested, maxReqCount)
   149  			}
   150  		}
   151  	}
   152  	// Ensure the result channel is torn down after the last block
   153  	match, ok := <-matches
   154  	if ok {
   155  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
   156  	}
   157  	// Clean up the session and ensure we match the expected retrieval count
   158  	session.Close(time.Second)
   159  	close(quit)
   160  
   161  	if retrievals != 0 && requested != retrievals {
   162  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals)
   163  	}
   164  	return requested
   165  }
   166  
   167  // startRetrievers starts a batch of goroutines listening for section requests
   168  // and serving them.
   169  func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) {
   170  	requests := make(chan chan *Retrieval)
   171  
   172  	for i := 0; i < 10; i++ {
   173  		// Start a multiplexer to test multiple threaded execution
   174  		go session.Multiplex(batch, 100*time.Microsecond, requests)
   175  
   176  		// Start a services to match the above multiplexer
   177  		go func() {
   178  			for {
   179  				// Wait for a service request or a shutdown
   180  				select {
   181  				case <-quit:
   182  					return
   183  
   184  				case request := <-requests:
   185  					task := <-request
   186  
   187  					task.Bitsets = make([][]byte, len(task.Sections))
   188  					for i, section := range task.Sections {
   189  						if rand.Int()%4 != 0 { // Handle occasional missing deliveries
   190  							task.Bitsets[i] = generateBitset(task.Bit, section)
   191  							atomic.AddUint32(retrievals, 1)
   192  						}
   193  					}
   194  					request <- task
   195  				}
   196  			}
   197  		}()
   198  	}
   199  }
   200  
   201  // generateBitset generates the rotated bitset for the given bloom bit and section
   202  // numbers.
   203  func generateBitset(bit uint, section uint64) []byte {
   204  	bitset := make([]byte, testSectionSize/8)
   205  	for i := 0; i < len(bitset); i++ {
   206  		for b := 0; b < 8; b++ {
   207  			blockIdx := section*testSectionSize + uint64(i*8+b)
   208  			bitset[i] += bitset[i]
   209  			if (blockIdx % uint64(bit)) == 0 {
   210  				bitset[i]++
   211  			}
   212  		}
   213  	}
   214  	return bitset
   215  }
   216  
   217  func expMatch1(filter bloomIndexes, i uint64) bool {
   218  	for _, ii := range filter {
   219  		if (i % uint64(ii)) != 0 {
   220  			return false
   221  		}
   222  	}
   223  	return true
   224  }
   225  
   226  func expMatch2(filter []bloomIndexes, i uint64) bool {
   227  	for _, ii := range filter {
   228  		if expMatch1(ii, i) {
   229  			return true
   230  		}
   231  	}
   232  	return false
   233  }
   234  
   235  func expMatch3(filter [][]bloomIndexes, i uint64) bool {
   236  	for _, ii := range filter {
   237  		if !expMatch2(ii, i) {
   238  			return false
   239  		}
   240  	}
   241  	return true
   242  }