github.com/arieschain/arieschain@v0.0.0-20191023063405-37c074544356/core/bloombits/matcher_test.go (about)

     1  package bloombits
     2  
     3  import (
     4  	"context"
     5  	"math/rand"
     6  	"sync/atomic"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/quickchainproject/quickchain/common"
    11  )
    12  
    13  const testSectionSize = 4096
    14  
    15  // Tests that wildcard filter rules (nil) can be specified and are handled well.
    16  func TestMatcherWildcards(t *testing.T) {
    17  	matcher := NewMatcher(testSectionSize, [][][]byte{
    18  		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
    19  		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
    20  		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
    21  		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
    22  		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
    23  		{nil, nil},                                               // Wildcard combo, drop rule
    24  		{},                                                       // Inited wildcard rule, drop rule
    25  		nil,                                                      // Proper wildcard rule, drop rule
    26  	})
    27  	if len(matcher.filters) != 3 {
    28  		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
    29  	}
    30  	if len(matcher.filters[0]) != 2 {
    31  		t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2)
    32  	}
    33  	if len(matcher.filters[1]) != 2 {
    34  		t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2)
    35  	}
    36  	if len(matcher.filters[2]) != 1 {
    37  		t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1)
    38  	}
    39  }
    40  
    41  // Tests the matcher pipeline on a single continuous workflow without interrupts.
    42  func TestMatcherContinuous(t *testing.T) {
    43  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
    44  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
    45  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
    46  }
    47  
    48  // Tests the matcher pipeline on a constantly interrupted and resumed work pattern
    49  // with the aim of ensuring data items are requested only once.
    50  func TestMatcherIntermittent(t *testing.T) {
    51  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
    52  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
    53  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
    54  }
    55  
    56  // Tests the matcher pipeline on random input to hopefully catch anomalies.
    57  func TestMatcherRandom(t *testing.T) {
    58  	for i := 0; i < 10; i++ {
    59  		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
    60  		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
    61  		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
    62  		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
    63  		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
    64  	}
    65  }
    66  
    67  // Tests that the matcher can properly find matches if the starting block is
    68  // shifter from a multiple of 8. This is needed to cover an optimisation with
    69  // bitset matching https://github.com/quickchainproject/quickchain/issues/15309.
    70  func TestMatcherShifted(t *testing.T) {
    71  	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
    72  	// start to get a potential zero byte in the matcher bitset.
    73  
    74  	// To keep the second bitset byte zero, the filter must only match for the first
    75  	// time in block 16, so doing an all-16 bit filter should suffice.
    76  
    77  	// To keep the starting block non divisible by 8, block number 9 is the first
    78  	// that would introduce a shift and not match block 0.
    79  	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
    80  }
    81  
    82  // Tests that matching on everything doesn't crash (special case internally).
    83  func TestWildcardMatcher(t *testing.T) {
    84  	testMatcherBothModes(t, nil, 0, 10000, 0)
    85  }
    86  
    87  // makeRandomIndexes generates a random filter system, composed on multiple filter
    88  // criteria, each having one bloom list component for the address and arbitrarily
    89  // many topic bloom list components.
    90  func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
    91  	res := make([][]bloomIndexes, len(lengths))
    92  	for i, topics := range lengths {
    93  		res[i] = make([]bloomIndexes, topics)
    94  		for j := 0; j < topics; j++ {
    95  			for k := 0; k < len(res[i][j]); k++ {
    96  				res[i][j][k] = uint(rand.Intn(max-1) + 2)
    97  			}
    98  		}
    99  	}
   100  	return res
   101  }
   102  
   103  // testMatcherDiffBatches runs the given matches test in single-delivery and also
   104  // in batches delivery mode, verifying that all kinds of deliveries are handled
   105  // correctly withn.
   106  func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
   107  	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
   108  	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
   109  
   110  	if singleton != batched {
   111  		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
   112  	}
   113  }
   114  
   115  // testMatcherBothModes runs the given matcher test in both continuous as well as
   116  // in intermittent mode, verifying that the request counts match each other.
   117  func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
   118  	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
   119  	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
   120  
   121  	if continuous != intermittent {
   122  		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
   123  	}
   124  }
   125  
   126  // testMatcher is a generic tester to run the given matcher test and return the
   127  // number of requests made for cross validation between different modes.
   128  func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
   129  	// Create a new matcher an simulate our explicit random bitsets
   130  	matcher := NewMatcher(testSectionSize, nil)
   131  	matcher.filters = filter
   132  
   133  	for _, rule := range filter {
   134  		for _, topic := range rule {
   135  			for _, bit := range topic {
   136  				matcher.addScheduler(bit)
   137  			}
   138  		}
   139  	}
   140  	// Track the number of retrieval requests made
   141  	var requested uint32
   142  
   143  	// Start the matching session for the filter and the retriver goroutines
   144  	quit := make(chan struct{})
   145  	matches := make(chan uint64, 16)
   146  
   147  	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
   148  	if err != nil {
   149  		t.Fatalf("failed to stat matcher session: %v", err)
   150  	}
   151  	startRetrievers(session, quit, &requested, maxReqCount)
   152  
   153  	// Iterate over all the blocks and verify that the pipeline produces the correct matches
   154  	for i := start; i < blocks; i++ {
   155  		if expMatch3(filter, i) {
   156  			match, ok := <-matches
   157  			if !ok {
   158  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
   159  				return 0
   160  			}
   161  			if match != i {
   162  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
   163  			}
   164  			// If we're testing intermittent mode, abort and restart the pipeline
   165  			if intermittent {
   166  				session.Close()
   167  				close(quit)
   168  
   169  				quit = make(chan struct{})
   170  				matches = make(chan uint64, 16)
   171  
   172  				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
   173  				if err != nil {
   174  					t.Fatalf("failed to stat matcher session: %v", err)
   175  				}
   176  				startRetrievers(session, quit, &requested, maxReqCount)
   177  			}
   178  		}
   179  	}
   180  	// Ensure the result channel is torn down after the last block
   181  	match, ok := <-matches
   182  	if ok {
   183  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
   184  	}
   185  	// Clean up the session and ensure we match the expected retrieval count
   186  	session.Close()
   187  	close(quit)
   188  
   189  	if retrievals != 0 && requested != retrievals {
   190  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals)
   191  	}
   192  	return requested
   193  }
   194  
   195  // startRetrievers starts a batch of goroutines listening for section requests
   196  // and serving them.
   197  func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) {
   198  	requests := make(chan chan *Retrieval)
   199  
   200  	for i := 0; i < 10; i++ {
   201  		// Start a multiplexer to test multiple threaded execution
   202  		go session.Multiplex(batch, 100*time.Microsecond, requests)
   203  
   204  		// Start a services to match the above multiplexer
   205  		go func() {
   206  			for {
   207  				// Wait for a service request or a shutdown
   208  				select {
   209  				case <-quit:
   210  					return
   211  
   212  				case request := <-requests:
   213  					task := <-request
   214  
   215  					task.Bitsets = make([][]byte, len(task.Sections))
   216  					for i, section := range task.Sections {
   217  						if rand.Int()%4 != 0 { // Handle occasional missing deliveries
   218  							task.Bitsets[i] = generateBitset(task.Bit, section)
   219  							atomic.AddUint32(retrievals, 1)
   220  						}
   221  					}
   222  					request <- task
   223  				}
   224  			}
   225  		}()
   226  	}
   227  }
   228  
   229  // generateBitset generates the rotated bitset for the given bloom bit and section
   230  // numbers.
   231  func generateBitset(bit uint, section uint64) []byte {
   232  	bitset := make([]byte, testSectionSize/8)
   233  	for i := 0; i < len(bitset); i++ {
   234  		for b := 0; b < 8; b++ {
   235  			blockIdx := section*testSectionSize + uint64(i*8+b)
   236  			bitset[i] += bitset[i]
   237  			if (blockIdx % uint64(bit)) == 0 {
   238  				bitset[i]++
   239  			}
   240  		}
   241  	}
   242  	return bitset
   243  }
   244  
   245  func expMatch1(filter bloomIndexes, i uint64) bool {
   246  	for _, ii := range filter {
   247  		if (i % uint64(ii)) != 0 {
   248  			return false
   249  		}
   250  	}
   251  	return true
   252  }
   253  
   254  func expMatch2(filter []bloomIndexes, i uint64) bool {
   255  	for _, ii := range filter {
   256  		if expMatch1(ii, i) {
   257  			return true
   258  		}
   259  	}
   260  	return false
   261  }
   262  
   263  func expMatch3(filter [][]bloomIndexes, i uint64) bool {
   264  	for _, ii := range filter {
   265  		if !expMatch2(ii, i) {
   266  			return false
   267  		}
   268  	}
   269  	return true
   270  }