github.com/YoungNK/go-ethereum@v1.9.7/core/bloombits/matcher_test.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package bloombits
    18  
    19  import (
    20  	"context"
    21  	"math/rand"
    22  	"sync/atomic"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/ethereum/go-ethereum/common"
    27  )
    28  
    29  const testSectionSize = 4096
    30  
    31  // Tests that wildcard filter rules (nil) can be specified and are handled well.
    32  func TestMatcherWildcards(t *testing.T) {
    33  	matcher := NewMatcher(testSectionSize, [][][]byte{
    34  		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
    35  		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
    36  		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
    37  		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
    38  		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
    39  		{nil, nil},                                               // Wildcard combo, drop rule
    40  		{},                                                       // Inited wildcard rule, drop rule
    41  		nil,                                                      // Proper wildcard rule, drop rule
    42  	})
    43  	if len(matcher.filters) != 3 {
    44  		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
    45  	}
    46  	if len(matcher.filters[0]) != 2 {
    47  		t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2)
    48  	}
    49  	if len(matcher.filters[1]) != 2 {
    50  		t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2)
    51  	}
    52  	if len(matcher.filters[2]) != 1 {
    53  		t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1)
    54  	}
    55  }
    56  
    57  // Tests the matcher pipeline on a single continuous workflow without interrupts.
    58  func TestMatcherContinuous(t *testing.T) {
    59  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
    60  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
    61  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
    62  }
    63  
    64  // Tests the matcher pipeline on a constantly interrupted and resumed work pattern
    65  // with the aim of ensuring data items are requested only once.
    66  func TestMatcherIntermittent(t *testing.T) {
    67  	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
    68  	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
    69  	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
    70  }
    71  
    72  // Tests the matcher pipeline on random input to hopefully catch anomalies.
    73  func TestMatcherRandom(t *testing.T) {
    74  	for i := 0; i < 10; i++ {
    75  		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
    76  		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
    77  		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
    78  		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
    79  		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
    80  	}
    81  }
    82  
    83  // Tests that the matcher can properly find matches if the starting block is
    84  // shifter from a multiple of 8. This is needed to cover an optimisation with
    85  // bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
    86  func TestMatcherShifted(t *testing.T) {
    87  	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
    88  	// start to get a potential zero byte in the matcher bitset.
    89  
    90  	// To keep the second bitset byte zero, the filter must only match for the first
    91  	// time in block 16, so doing an all-16 bit filter should suffice.
    92  
    93  	// To keep the starting block non divisible by 8, block number 9 is the first
    94  	// that would introduce a shift and not match block 0.
    95  	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
    96  }
    97  
    98  // Tests that matching on everything doesn't crash (special case internally).
    99  func TestWildcardMatcher(t *testing.T) {
   100  	testMatcherBothModes(t, nil, 0, 10000, 0)
   101  }
   102  
   103  // makeRandomIndexes generates a random filter system, composed on multiple filter
   104  // criteria, each having one bloom list component for the address and arbitrarily
   105  // many topic bloom list components.
   106  func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
   107  	res := make([][]bloomIndexes, len(lengths))
   108  	for i, topics := range lengths {
   109  		res[i] = make([]bloomIndexes, topics)
   110  		for j := 0; j < topics; j++ {
   111  			for k := 0; k < len(res[i][j]); k++ {
   112  				res[i][j][k] = uint(rand.Intn(max-1) + 2)
   113  			}
   114  		}
   115  	}
   116  	return res
   117  }
   118  
   119  // testMatcherDiffBatches runs the given matches test in single-delivery and also
   120  // in batches delivery mode, verifying that all kinds of deliveries are handled
   121  // correctly withn.
   122  func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
   123  	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
   124  	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
   125  
   126  	if singleton != batched {
   127  		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
   128  	}
   129  }
   130  
   131  // testMatcherBothModes runs the given matcher test in both continuous as well as
   132  // in intermittent mode, verifying that the request counts match each other.
   133  func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
   134  	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
   135  	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
   136  
   137  	if continuous != intermittent {
   138  		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
   139  	}
   140  }
   141  
   142  // testMatcher is a generic tester to run the given matcher test and return the
   143  // number of requests made for cross validation between different modes.
   144  func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
   145  	// Create a new matcher an simulate our explicit random bitsets
   146  	matcher := NewMatcher(testSectionSize, nil)
   147  	matcher.filters = filter
   148  
   149  	for _, rule := range filter {
   150  		for _, topic := range rule {
   151  			for _, bit := range topic {
   152  				matcher.addScheduler(bit)
   153  			}
   154  		}
   155  	}
   156  	// Track the number of retrieval requests made
   157  	var requested uint32
   158  
   159  	// Start the matching session for the filter and the retriever goroutines
   160  	quit := make(chan struct{})
   161  	matches := make(chan uint64, 16)
   162  
   163  	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
   164  	if err != nil {
   165  		t.Fatalf("failed to stat matcher session: %v", err)
   166  	}
   167  	startRetrievers(session, quit, &requested, maxReqCount)
   168  
   169  	// Iterate over all the blocks and verify that the pipeline produces the correct matches
   170  	for i := start; i < blocks; i++ {
   171  		if expMatch3(filter, i) {
   172  			match, ok := <-matches
   173  			if !ok {
   174  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
   175  				return 0
   176  			}
   177  			if match != i {
   178  				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
   179  			}
   180  			// If we're testing intermittent mode, abort and restart the pipeline
   181  			if intermittent {
   182  				session.Close()
   183  				close(quit)
   184  
   185  				quit = make(chan struct{})
   186  				matches = make(chan uint64, 16)
   187  
   188  				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
   189  				if err != nil {
   190  					t.Fatalf("failed to stat matcher session: %v", err)
   191  				}
   192  				startRetrievers(session, quit, &requested, maxReqCount)
   193  			}
   194  		}
   195  	}
   196  	// Ensure the result channel is torn down after the last block
   197  	match, ok := <-matches
   198  	if ok {
   199  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
   200  	}
   201  	// Clean up the session and ensure we match the expected retrieval count
   202  	session.Close()
   203  	close(quit)
   204  
   205  	if retrievals != 0 && requested != retrievals {
   206  		t.Errorf("filter = %v  blocks = %v  intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals)
   207  	}
   208  	return requested
   209  }
   210  
   211  // startRetrievers starts a batch of goroutines listening for section requests
   212  // and serving them.
   213  func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) {
   214  	requests := make(chan chan *Retrieval)
   215  
   216  	for i := 0; i < 10; i++ {
   217  		// Start a multiplexer to test multiple threaded execution
   218  		go session.Multiplex(batch, 100*time.Microsecond, requests)
   219  
   220  		// Start a services to match the above multiplexer
   221  		go func() {
   222  			for {
   223  				// Wait for a service request or a shutdown
   224  				select {
   225  				case <-quit:
   226  					return
   227  
   228  				case request := <-requests:
   229  					task := <-request
   230  
   231  					task.Bitsets = make([][]byte, len(task.Sections))
   232  					for i, section := range task.Sections {
   233  						if rand.Int()%4 != 0 { // Handle occasional missing deliveries
   234  							task.Bitsets[i] = generateBitset(task.Bit, section)
   235  							atomic.AddUint32(retrievals, 1)
   236  						}
   237  					}
   238  					request <- task
   239  				}
   240  			}
   241  		}()
   242  	}
   243  }
   244  
   245  // generateBitset generates the rotated bitset for the given bloom bit and section
   246  // numbers.
   247  func generateBitset(bit uint, section uint64) []byte {
   248  	bitset := make([]byte, testSectionSize/8)
   249  	for i := 0; i < len(bitset); i++ {
   250  		for b := 0; b < 8; b++ {
   251  			blockIdx := section*testSectionSize + uint64(i*8+b)
   252  			bitset[i] += bitset[i]
   253  			if (blockIdx % uint64(bit)) == 0 {
   254  				bitset[i]++
   255  			}
   256  		}
   257  	}
   258  	return bitset
   259  }
   260  
   261  func expMatch1(filter bloomIndexes, i uint64) bool {
   262  	for _, ii := range filter {
   263  		if (i % uint64(ii)) != 0 {
   264  			return false
   265  		}
   266  	}
   267  	return true
   268  }
   269  
   270  func expMatch2(filter []bloomIndexes, i uint64) bool {
   271  	for _, ii := range filter {
   272  		if expMatch1(ii, i) {
   273  			return true
   274  		}
   275  	}
   276  	return false
   277  }
   278  
   279  func expMatch3(filter [][]bloomIndexes, i uint64) bool {
   280  	for _, ii := range filter {
   281  		if !expMatch2(ii, i) {
   282  			return false
   283  		}
   284  	}
   285  	return true
   286  }