github.com/koko1123/flow-go-1@v0.29.6/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go (about)

     1  package votecollector
     2  
     3  import (
     4  	"errors"
     5  	"math/rand"
     6  	"sync"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/stretchr/testify/mock"
    11  	"github.com/stretchr/testify/require"
    12  	"github.com/stretchr/testify/suite"
    13  	"go.uber.org/atomic"
    14  	"pgregory.net/rapid"
    15  
    16  	bootstrapDKG "github.com/koko1123/flow-go-1/cmd/bootstrap/dkg"
    17  	"github.com/koko1123/flow-go-1/consensus/hotstuff"
    18  	"github.com/koko1123/flow-go-1/consensus/hotstuff/helper"
    19  	mockhotstuff "github.com/koko1123/flow-go-1/consensus/hotstuff/mocks"
    20  	"github.com/koko1123/flow-go-1/consensus/hotstuff/model"
    21  	hsig "github.com/koko1123/flow-go-1/consensus/hotstuff/signature"
    22  	hotstuffvalidator "github.com/koko1123/flow-go-1/consensus/hotstuff/validator"
    23  	"github.com/koko1123/flow-go-1/consensus/hotstuff/verification"
    24  	"github.com/koko1123/flow-go-1/model/encodable"
    25  	"github.com/koko1123/flow-go-1/model/flow"
    26  	"github.com/koko1123/flow-go-1/module/local"
    27  	modulemock "github.com/koko1123/flow-go-1/module/mock"
    28  	"github.com/koko1123/flow-go-1/module/signature"
    29  	"github.com/koko1123/flow-go-1/state/protocol/inmem"
    30  	storagemock "github.com/koko1123/flow-go-1/storage/mock"
    31  	"github.com/koko1123/flow-go-1/utils/unittest"
    32  	"github.com/onflow/flow-go/crypto"
    33  )
    34  
    35  func TestCombinedVoteProcessorV3(t *testing.T) {
    36  	suite.Run(t, new(CombinedVoteProcessorV3TestSuite))
    37  }
    38  
    39  // CombinedVoteProcessorV3TestSuite is a test suite that holds mocked state for isolated testing of CombinedVoteProcessorV3.
    40  type CombinedVoteProcessorV3TestSuite struct {
    41  	VoteProcessorTestSuiteBase
    42  
    43  	thresholdTotalWeight atomic.Uint64
    44  	rbSharesTotal        atomic.Uint64
    45  
    46  	packer *mockhotstuff.Packer
    47  
    48  	rbSigAggregator *mockhotstuff.WeightedSignatureAggregator
    49  	reconstructor   *mockhotstuff.RandomBeaconReconstructor
    50  
    51  	minRequiredShares uint64
    52  	processor         *CombinedVoteProcessorV3
    53  }
    54  
    55  func (s *CombinedVoteProcessorV3TestSuite) SetupTest() {
    56  	s.VoteProcessorTestSuiteBase.SetupTest()
    57  
    58  	s.rbSigAggregator = &mockhotstuff.WeightedSignatureAggregator{}
    59  	s.reconstructor = &mockhotstuff.RandomBeaconReconstructor{}
    60  	s.packer = &mockhotstuff.Packer{}
    61  	s.proposal = helper.MakeProposal()
    62  
    63  	s.minRequiredShares = 9 // we require 9 RB shares to reconstruct signature
    64  	s.thresholdTotalWeight, s.rbSharesTotal = atomic.Uint64{}, atomic.Uint64{}
    65  
    66  	// setup threshold signature aggregator
    67  	s.rbSigAggregator.On("TrustedAdd", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
    68  		s.thresholdTotalWeight.Add(s.sigWeight)
    69  	}).Return(func(signerID flow.Identifier, sig crypto.Signature) uint64 {
    70  		return s.thresholdTotalWeight.Load()
    71  	}, func(signerID flow.Identifier, sig crypto.Signature) error {
    72  		return nil
    73  	}).Maybe()
    74  	s.rbSigAggregator.On("TotalWeight").Return(func() uint64 {
    75  		return s.thresholdTotalWeight.Load()
    76  	}).Maybe()
    77  
    78  	// setup rb reconstructor
    79  	s.reconstructor.On("TrustedAdd", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
    80  		s.rbSharesTotal.Inc()
    81  	}).Return(func(signerID flow.Identifier, sig crypto.Signature) bool {
    82  		return s.rbSharesTotal.Load() >= s.minRequiredShares
    83  	}, func(signerID flow.Identifier, sig crypto.Signature) error {
    84  		return nil
    85  	}).Maybe()
    86  	s.reconstructor.On("EnoughShares").Return(func() bool {
    87  		return s.rbSharesTotal.Load() >= s.minRequiredShares
    88  	}).Maybe()
    89  
    90  	s.processor = &CombinedVoteProcessorV3{
    91  		log:               unittest.Logger(),
    92  		block:             s.proposal.Block,
    93  		stakingSigAggtor:  s.stakingAggregator,
    94  		rbSigAggtor:       s.rbSigAggregator,
    95  		rbRector:          s.reconstructor,
    96  		onQCCreated:       s.onQCCreated,
    97  		packer:            s.packer,
    98  		minRequiredWeight: s.minRequiredWeight,
    99  		done:              *atomic.NewBool(false),
   100  	}
   101  }
   102  
   103  // TestInitialState tests that Block() and Status() return correct values after calling constructor
   104  func (s *CombinedVoteProcessorV3TestSuite) TestInitialState() {
   105  	require.Equal(s.T(), s.proposal.Block, s.processor.Block())
   106  	require.Equal(s.T(), hotstuff.VoteCollectorStatusVerifying, s.processor.Status())
   107  }
   108  
   109  // TestProcess_VoteNotForProposal tests that CombinedVoteProcessorV3 accepts only votes for the block it was initialized with
   110  // according to interface specification of `VoteProcessor`, we expect dedicated sentinel errors for votes
   111  // for different views (`VoteForIncompatibleViewError`) _or_ block (`VoteForIncompatibleBlockError`).
   112  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_VoteNotForProposal() {
   113  	err := s.processor.Process(unittest.VoteFixture(unittest.WithVoteView(s.proposal.Block.View)))
   114  	require.ErrorAs(s.T(), err, &VoteForIncompatibleBlockError)
   115  	require.False(s.T(), model.IsInvalidVoteError(err))
   116  
   117  	err = s.processor.Process(unittest.VoteFixture(unittest.WithVoteBlockID(s.proposal.Block.BlockID)))
   118  	require.ErrorAs(s.T(), err, &VoteForIncompatibleViewError)
   119  	require.False(s.T(), model.IsInvalidVoteError(err))
   120  
   121  	s.stakingAggregator.AssertNotCalled(s.T(), "Verify")
   122  	s.rbSigAggregator.AssertNotCalled(s.T(), "Verify")
   123  }
   124  
   125  // TestProcess_InvalidSignatureFormat ensures that we process signatures only with valid format.
   126  // If we have received vote with signature in invalid format we should return with sentinel error
   127  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_InvalidSignatureFormat() {
   128  	// signature is random in this case
   129  	vote := unittest.VoteForBlockFixture(s.proposal.Block, func(vote *model.Vote) {
   130  		vote.SigData[0] = byte(42)
   131  	})
   132  	err := s.processor.Process(vote)
   133  	require.Error(s.T(), err)
   134  	require.True(s.T(), model.IsInvalidVoteError(err))
   135  	require.True(s.T(), errors.Is(err, signature.ErrInvalidSignatureFormat), err)
   136  }
   137  
   138  // TestProcess_InvalidSignature tests that CombinedVoteProcessorV2 rejects invalid votes for the following scenarios:
   139  //  1. vote containing staking sig
   140  //  2. vote containing random beacon sig
   141  //
   142  // For each scenario, we test two sub-cases:
   143  //   - `SignerID` is not a valid consensus participant;
   144  //   - `SignerID` is valid consensus participant but the signature is cryptographically invalid
   145  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_InvalidSignature() {
   146  	s.Run("vote with staking-sig", func() {
   147  		// sentinel error from `InvalidSignerError` should be wrapped as `InvalidVoteError`
   148  		voteA := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   149  		s.stakingAggregator.On("Verify", voteA.SignerID, mock.Anything).Return(model.NewInvalidSignerErrorf("")).Once()
   150  		err := s.processor.Process(voteA)
   151  		require.Error(s.T(), err)
   152  		require.True(s.T(), model.IsInvalidVoteError(err))
   153  		require.True(s.T(), model.IsInvalidSignerError(err))
   154  
   155  		// sentinel error from `ErrInvalidSignature` should be wrapped as `InvalidVoteError`
   156  		voteB := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   157  		s.stakingAggregator.On("Verify", voteB.SignerID, mock.Anything).Return(model.ErrInvalidSignature).Once()
   158  		err = s.processor.Process(voteB)
   159  		require.Error(s.T(), err)
   160  		require.True(s.T(), model.IsInvalidVoteError(err))
   161  		require.ErrorAs(s.T(), err, &model.ErrInvalidSignature)
   162  
   163  		s.stakingAggregator.AssertNotCalled(s.T(), "TrustedAdd")
   164  	})
   165  
   166  	s.Run("vote with beacon-sig", func() {
   167  		// sentinel error from `InvalidSignerError` should be wrapped as `InvalidVoteError`
   168  		voteA := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithBeaconSig())
   169  		s.rbSigAggregator.On("Verify", voteA.SignerID, mock.Anything).Return(model.NewInvalidSignerErrorf("")).Once()
   170  		err := s.processor.Process(voteA)
   171  		require.Error(s.T(), err)
   172  		require.True(s.T(), model.IsInvalidVoteError(err))
   173  		require.True(s.T(), model.IsInvalidSignerError(err))
   174  
   175  		// sentinel error from `ErrInvalidSignature` should be wrapped as `InvalidVoteError`
   176  		voteB := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithBeaconSig())
   177  		s.rbSigAggregator.On("Verify", voteB.SignerID, mock.Anything).Return(model.ErrInvalidSignature).Once()
   178  		err = s.processor.Process(voteB)
   179  		require.Error(s.T(), err)
   180  		require.True(s.T(), model.IsInvalidVoteError(err))
   181  		require.ErrorAs(s.T(), err, &model.ErrInvalidSignature)
   182  
   183  		s.rbSigAggregator.AssertNotCalled(s.T(), "TrustedAdd")
   184  		s.reconstructor.AssertNotCalled(s.T(), "TrustedAdd")
   185  	})
   186  
   187  }
   188  
   189  // TestProcess_TrustedAdd_Exception tests that unexpected exceptions returned by
   190  // WeightedSignatureAggregator.Verify(..) are propagated, but _not_ interpreted as invalid votes
   191  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_Verify_Exception() {
   192  	exception := errors.New("unexpected-exception")
   193  
   194  	s.Run("vote with staking-sig", func() {
   195  		stakingVote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   196  		s.stakingAggregator.On("Verify", stakingVote.SignerID, mock.Anything).Return(exception)
   197  
   198  		err := s.processor.Process(stakingVote)
   199  		require.ErrorIs(s.T(), err, exception)              // unexpected errors from verifying the vote signature should be propagated
   200  		require.False(s.T(), model.IsInvalidVoteError(err)) // but not interpreted as an invalid vote
   201  		s.stakingAggregator.AssertNotCalled(s.T(), "TrustedAdd")
   202  	})
   203  
   204  	s.Run("vote with beacon-sig", func() {
   205  		beaconVote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithBeaconSig())
   206  		s.rbSigAggregator.On("Verify", beaconVote.SignerID, mock.Anything).Return(exception)
   207  
   208  		err := s.processor.Process(beaconVote)
   209  		require.ErrorIs(s.T(), err, exception)              // unexpected errors from verifying the vote signature should be propagated
   210  		require.False(s.T(), model.IsInvalidVoteError(err)) // but not interpreted as an invalid vote
   211  		s.rbSigAggregator.AssertNotCalled(s.T(), "TrustedAdd")
   212  		s.reconstructor.AssertNotCalled(s.T(), "TrustedAdd")
   213  	})
   214  }
   215  
   216  // TestProcess_TrustedAdd_Exception tests that unexpected exceptions returned by
   217  // WeightedSignatureAggregator.TrustedAdd(..) are _not_ interpreted as invalid votes
   218  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_TrustedAdd_Exception() {
   219  	exception := errors.New("unexpected-exception")
   220  	s.Run("staking-sig", func() {
   221  		stakingVote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   222  		*s.stakingAggregator = mockhotstuff.WeightedSignatureAggregator{}
   223  		s.stakingAggregator.On("Verify", stakingVote.SignerID, mock.Anything).Return(nil).Once()
   224  		s.stakingAggregator.On("TrustedAdd", stakingVote.SignerID, mock.Anything).Return(uint64(0), exception).Once()
   225  		err := s.processor.Process(stakingVote)
   226  		require.ErrorIs(s.T(), err, exception)
   227  		require.False(s.T(), model.IsInvalidVoteError(err))
   228  	})
   229  	s.Run("threshold-sig", func() {
   230  		thresholdVote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithBeaconSig())
   231  		*s.rbSigAggregator = mockhotstuff.WeightedSignatureAggregator{}
   232  		*s.reconstructor = mockhotstuff.RandomBeaconReconstructor{}
   233  		s.rbSigAggregator.On("Verify", thresholdVote.SignerID, mock.Anything).Return(nil)
   234  		s.rbSigAggregator.On("TrustedAdd", thresholdVote.SignerID, mock.Anything).Return(uint64(0), exception).Once()
   235  		err := s.processor.Process(thresholdVote)
   236  		require.ErrorIs(s.T(), err, exception)
   237  		require.False(s.T(), model.IsInvalidVoteError(err))
   238  		// test also if reconstructor failed to add it
   239  		s.rbSigAggregator.On("TrustedAdd", thresholdVote.SignerID, mock.Anything).Return(s.sigWeight, nil).Once()
   240  		s.reconstructor.On("TrustedAdd", thresholdVote.SignerID, mock.Anything).Return(false, exception).Once()
   241  		err = s.processor.Process(thresholdVote)
   242  		require.ErrorIs(s.T(), err, exception)
   243  		require.False(s.T(), model.IsInvalidVoteError(err))
   244  	})
   245  }
   246  
   247  // TestProcess_BuildQCError tests all error paths during process of building QC.
   248  // Building QC is a one time operation, we need to make sure that failing in one of the steps leads to exception.
   249  // Since it's a one time operation we need a complicated test to test all conditions.
   250  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_BuildQCError() {
   251  	mockAggregator := func(aggregator *mockhotstuff.WeightedSignatureAggregator) {
   252  		aggregator.On("Verify", mock.Anything, mock.Anything).Return(nil)
   253  		aggregator.On("TrustedAdd", mock.Anything, mock.Anything).Return(s.minRequiredWeight, nil)
   254  		aggregator.On("TotalWeight").Return(s.minRequiredWeight)
   255  	}
   256  
   257  	stakingSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   258  	thresholdSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   259  	reconstructor := &mockhotstuff.RandomBeaconReconstructor{}
   260  	packer := &mockhotstuff.Packer{}
   261  
   262  	identities := unittest.IdentifierListFixture(5)
   263  
   264  	// In this test we will mock all dependencies for happy path, and replace some branches with unhappy path
   265  	// to simulate errors along the branches.
   266  
   267  	mockAggregator(stakingSigAggregator)
   268  	stakingSigAggregator.On("Aggregate").Return(identities, unittest.RandomBytes(128), nil)
   269  
   270  	mockAggregator(thresholdSigAggregator)
   271  	thresholdSigAggregator.On("Aggregate").Return(identities, unittest.RandomBytes(128), nil)
   272  
   273  	reconstructor.On("EnoughShares").Return(true)
   274  	reconstructor.On("Reconstruct").Return(unittest.SignatureFixture(), nil)
   275  
   276  	packer.On("Pack", mock.Anything, mock.Anything).Return(identities, unittest.RandomBytes(128), nil)
   277  
   278  	// Helper factory function to create processors. We need new processor for every test case
   279  	// because QC creation is one time operation and is triggered as soon as we have collected enough weight and shares.
   280  	createProcessor := func(stakingAggregator *mockhotstuff.WeightedSignatureAggregator,
   281  		rbSigAggregator *mockhotstuff.WeightedSignatureAggregator,
   282  		rbReconstructor *mockhotstuff.RandomBeaconReconstructor,
   283  		packer *mockhotstuff.Packer) *CombinedVoteProcessorV3 {
   284  		return &CombinedVoteProcessorV3{
   285  			log:               unittest.Logger(),
   286  			block:             s.proposal.Block,
   287  			stakingSigAggtor:  stakingAggregator,
   288  			rbSigAggtor:       rbSigAggregator,
   289  			rbRector:          rbReconstructor,
   290  			onQCCreated:       s.onQCCreated,
   291  			packer:            packer,
   292  			minRequiredWeight: s.minRequiredWeight,
   293  			done:              *atomic.NewBool(false),
   294  		}
   295  	}
   296  
   297  	vote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   298  
   299  	// in this test case we aren't able to aggregate staking signature
   300  	s.Run("staking-sig-aggregate", func() {
   301  		exception := errors.New("staking-aggregate-exception")
   302  		stakingSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   303  		mockAggregator(stakingSigAggregator)
   304  		stakingSigAggregator.On("Aggregate").Return(nil, nil, exception)
   305  		processor := createProcessor(stakingSigAggregator, thresholdSigAggregator, reconstructor, packer)
   306  		err := processor.Process(vote)
   307  		require.ErrorIs(s.T(), err, exception)
   308  		require.False(s.T(), model.IsInvalidVoteError(err))
   309  	})
   310  	// in this test case we aren't able to aggregate threshold signature
   311  	s.Run("threshold-sig-aggregate", func() {
   312  		exception := errors.New("threshold-aggregate-exception")
   313  		thresholdSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   314  		mockAggregator(thresholdSigAggregator)
   315  		thresholdSigAggregator.On("Aggregate").Return(nil, nil, exception)
   316  		processor := createProcessor(stakingSigAggregator, thresholdSigAggregator, reconstructor, packer)
   317  		err := processor.Process(vote)
   318  		require.ErrorIs(s.T(), err, exception)
   319  		require.False(s.T(), model.IsInvalidVoteError(err))
   320  	})
   321  	// in this test case we aren't able to reconstruct signature
   322  	s.Run("reconstruct", func() {
   323  		exception := errors.New("reconstruct-exception")
   324  		reconstructor := &mockhotstuff.RandomBeaconReconstructor{}
   325  		reconstructor.On("EnoughShares").Return(true)
   326  		reconstructor.On("Reconstruct").Return(nil, exception)
   327  		processor := createProcessor(stakingSigAggregator, thresholdSigAggregator, reconstructor, packer)
   328  		err := processor.Process(vote)
   329  		require.ErrorIs(s.T(), err, exception)
   330  		require.False(s.T(), model.IsInvalidVoteError(err))
   331  	})
   332  	// in this test case we aren't able to pack signatures
   333  	s.Run("pack", func() {
   334  		exception := errors.New("pack-qc-exception")
   335  		packer := &mockhotstuff.Packer{}
   336  		packer.On("Pack", mock.Anything, mock.Anything).Return(nil, nil, exception)
   337  		processor := createProcessor(stakingSigAggregator, thresholdSigAggregator, reconstructor, packer)
   338  		err := processor.Process(vote)
   339  		require.ErrorIs(s.T(), err, exception)
   340  		require.False(s.T(), model.IsInvalidVoteError(err))
   341  	})
   342  }
   343  
   344  // TestProcess_EnoughWeightNotEnoughShares tests a scenario where we first don't have enough weight,
   345  // then we iteratively increase it to the point where we have enough staking weight. No QC should be created
   346  // in this scenario since there is not enough random beacon shares.
   347  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_EnoughWeightNotEnoughShares() {
   348  	for i := uint64(0); i < s.minRequiredWeight; i += s.sigWeight {
   349  		vote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   350  		s.stakingAggregator.On("Verify", vote.SignerID, mock.Anything).Return(nil)
   351  		err := s.processor.Process(vote)
   352  		require.NoError(s.T(), err)
   353  	}
   354  
   355  	require.False(s.T(), s.processor.done.Load())
   356  	s.reconstructor.AssertCalled(s.T(), "EnoughShares")
   357  	s.onQCCreatedState.AssertNotCalled(s.T(), "onQCCreated")
   358  }
   359  
   360  // TestProcess_EnoughSharesNotEnoughWeight tests a scenario where we are collecting only threshold signatures
   361  // to the point where we have enough shares to reconstruct RB signature. No QC should be created
   362  // in this scenario since there is not enough staking weight.
   363  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_EnoughSharesNotEnoughWeight() {
   364  	// change sig weight to be really low, so we don't reach min staking weight while collecting
   365  	// threshold signatures
   366  	s.sigWeight = 10
   367  	for i := uint64(0); i < s.minRequiredShares; i++ {
   368  		vote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithBeaconSig())
   369  		s.rbSigAggregator.On("Verify", vote.SignerID, mock.Anything).Return(nil)
   370  		err := s.processor.Process(vote)
   371  		require.NoError(s.T(), err)
   372  	}
   373  
   374  	require.False(s.T(), s.processor.done.Load())
   375  	s.reconstructor.AssertNotCalled(s.T(), "EnoughShares")
   376  	s.onQCCreatedState.AssertNotCalled(s.T(), "onQCCreated")
   377  	// verify if we indeed have enough shares
   378  	require.True(s.T(), s.reconstructor.EnoughShares())
   379  }
   380  
   381  // TestProcess_ConcurrentCreatingQC tests a scenario where multiple goroutines process vote at same time,
   382  // we expect only one QC created in this scenario.
   383  func (s *CombinedVoteProcessorV3TestSuite) TestProcess_ConcurrentCreatingQC() {
   384  	stakingSigners := unittest.IdentifierListFixture(10)
   385  	mockAggregator := func(aggregator *mockhotstuff.WeightedSignatureAggregator) {
   386  		aggregator.On("Verify", mock.Anything, mock.Anything).Return(nil)
   387  		aggregator.On("TrustedAdd", mock.Anything, mock.Anything).Return(s.minRequiredWeight, nil)
   388  		aggregator.On("TotalWeight").Return(s.minRequiredWeight)
   389  		aggregator.On("Aggregate").Return(stakingSigners, unittest.RandomBytes(128), nil)
   390  	}
   391  
   392  	// mock aggregators, so we have enough weight and shares for creating QC
   393  	*s.stakingAggregator = mockhotstuff.WeightedSignatureAggregator{}
   394  	mockAggregator(s.stakingAggregator)
   395  	*s.rbSigAggregator = mockhotstuff.WeightedSignatureAggregator{}
   396  	mockAggregator(s.rbSigAggregator)
   397  	*s.reconstructor = mockhotstuff.RandomBeaconReconstructor{}
   398  	s.reconstructor.On("Reconstruct").Return(unittest.SignatureFixture(), nil)
   399  	s.reconstructor.On("EnoughShares").Return(true)
   400  
   401  	// at this point sending any vote should result in creating QC.
   402  	s.packer.On("Pack", s.proposal.Block.BlockID, mock.Anything).Return(unittest.RandomBytes(100), unittest.RandomBytes(128), nil)
   403  	s.onQCCreatedState.On("onQCCreated", mock.Anything).Return(nil).Once()
   404  
   405  	var startupWg, shutdownWg sync.WaitGroup
   406  
   407  	vote := unittest.VoteForBlockFixture(s.proposal.Block, unittest.VoteWithStakingSig())
   408  	startupWg.Add(1)
   409  	// prepare goroutines, so they are ready to submit a vote at roughly same time
   410  	for i := 0; i < 5; i++ {
   411  		shutdownWg.Add(1)
   412  		go func() {
   413  			defer shutdownWg.Done()
   414  			startupWg.Wait()
   415  			err := s.processor.Process(vote)
   416  			require.NoError(s.T(), err)
   417  		}()
   418  	}
   419  
   420  	startupWg.Done()
   421  
   422  	// wait for all routines to finish
   423  	shutdownWg.Wait()
   424  
   425  	s.onQCCreatedState.AssertNumberOfCalls(s.T(), "onQCCreated", 1)
   426  }
   427  
   428  // TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness uses property testing to test correctness of concurrent votes processing.
   429  // We randomly draw a committee with some number of staking, random beacon and byzantine nodes.
   430  // Values are drawn in a way that 1 <= honestParticipants <= participants <= maxParticipants
   431  // In each test iteration we expect to create a valid QC with all provided data as part of constructed QC.
   432  func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing.T) {
   433  	maxParticipants := uint64(53)
   434  
   435  	rapid.Check(testifyT, func(t *rapid.T) {
   436  		// draw participants in range 1 <= participants <= maxParticipants
   437  		participants := rapid.Uint64Range(1, maxParticipants).Draw(t, "participants").(uint64)
   438  		beaconSignersCount := rapid.Uint64Range(participants/2+1, participants).Draw(t, "beaconSigners").(uint64)
   439  		stakingSignersCount := participants - beaconSignersCount
   440  		require.Equal(t, participants, stakingSignersCount+beaconSignersCount)
   441  
   442  		// setup how many votes we need to create a QC
   443  		// 1 <= honestParticipants <= participants <= maxParticipants
   444  		honestParticipants := participants*2/3 + 1
   445  		sigWeight := uint64(100)
   446  		minRequiredWeight := honestParticipants * sigWeight
   447  
   448  		// proposing block
   449  		block := helper.MakeBlock()
   450  
   451  		t.Logf("running conf\n\t"+
   452  			"staking signers: %v, beacon signers: %v\n\t"+
   453  			"required weight: %v", stakingSignersCount, beaconSignersCount, minRequiredWeight)
   454  
   455  		stakingTotalWeight, thresholdTotalWeight, collectedShares := uint64(0), uint64(0), uint64(0)
   456  
   457  		// setup aggregators and reconstructor
   458  		stakingAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   459  		rbSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   460  		reconstructor := &mockhotstuff.RandomBeaconReconstructor{}
   461  
   462  		stakingSigners := unittest.IdentifierListFixture(int(stakingSignersCount))
   463  		beaconSigners := unittest.IdentifierListFixture(int(beaconSignersCount))
   464  
   465  		// lists to track signers that actually contributed their signatures
   466  		var (
   467  			aggregatedStakingSigners flow.IdentifierList
   468  			aggregatedBeaconSigners  flow.IdentifierList
   469  		)
   470  
   471  		// need separate locks to safely update vectors of voted signers
   472  		stakingAggregatorLock := &sync.Mutex{}
   473  		beaconAggregatorLock := &sync.Mutex{}
   474  		beaconReconstructorLock := &sync.Mutex{}
   475  
   476  		stakingAggregator.On("TotalWeight").Return(func() uint64 {
   477  			stakingAggregatorLock.Lock()
   478  			defer stakingAggregatorLock.Unlock()
   479  			return stakingTotalWeight
   480  		})
   481  		rbSigAggregator.On("TotalWeight").Return(func() uint64 {
   482  			beaconAggregatorLock.Lock()
   483  			defer beaconAggregatorLock.Unlock()
   484  			return thresholdTotalWeight
   485  		})
   486  		reconstructor.On("EnoughShares").Return(func() bool {
   487  			beaconReconstructorLock.Lock()
   488  			defer beaconReconstructorLock.Unlock()
   489  			return collectedShares >= beaconSignersCount
   490  		})
   491  
   492  		// mock expected calls to aggregators and reconstructor
   493  		combinedSigs := unittest.SignaturesFixture(3)
   494  		stakingAggregator.On("Aggregate").Return(
   495  			func() flow.IdentifierList {
   496  				stakingAggregatorLock.Lock()
   497  				defer stakingAggregatorLock.Unlock()
   498  				return aggregatedStakingSigners
   499  			},
   500  			func() []byte { return combinedSigs[0] },
   501  			func() error { return nil }).Maybe() // Aggregate is only called, if some staking sigs were collected
   502  
   503  		rbSigAggregator.On("Aggregate").Return(
   504  			func() flow.IdentifierList {
   505  				beaconAggregatorLock.Lock()
   506  				defer beaconAggregatorLock.Unlock()
   507  				return aggregatedBeaconSigners
   508  			},
   509  			func() []byte { return combinedSigs[1] },
   510  			func() error { return nil }).Once()
   511  		reconstructor.On("Reconstruct").Return(combinedSigs[2], nil).Once()
   512  
   513  		// mock expected call to Packer
   514  		mergedSignerIDs := (flow.IdentifierList)(nil)
   515  		packedSigData := unittest.RandomBytes(128)
   516  		pcker := &mockhotstuff.Packer{}
   517  		pcker.On("Pack", block.BlockID, mock.Anything).Run(func(args mock.Arguments) {
   518  			blockSigData := args.Get(1).(*hotstuff.BlockSignatureData)
   519  			// in the following, we check validity for each field of `blockSigData` individually
   520  
   521  			// 1. CHECK: `StakingSigners` and `RandomBeaconSigners`
   522  			// Verify that input `hotstuff.BlockSignatureData` has the expected structure.
   523  			//  * When the Vote Processor notices that constructing a valid QC is possible, it does
   524  			//    so with the signatures collected at this time.
   525  			//  * However, due to concurrency, additional votes might have been added to the aggregators
   526  			//    by tailing threads, _before_ we reach this validation logic. Therefore, the set of
   527  			//    signers in the aggregators might now be _larger_ than what is reported in the QC.
   528  			// Therefore, we test that the signers reported in the QC are a _subset_ of the signatures
   529  			// that are now in the aggregators.
   530  			// check that aggregated signers are part of all votes signers
   531  			// due to concurrent processing it is possible that Aggregate will return less that we have actually aggregated
   532  			// but still enough to construct the QC
   533  			stakingAggregatorLock.Lock()
   534  			require.Subset(t, aggregatedStakingSigners, blockSigData.StakingSigners)
   535  			stakingAggregatorLock.Unlock()
   536  
   537  			beaconAggregatorLock.Lock()
   538  			require.Subset(t, aggregatedBeaconSigners, blockSigData.RandomBeaconSigners)
   539  			beaconAggregatorLock.Unlock()
   540  
   541  			// 2. CHECK: supermajority
   542  			// All participants have equal weights in this test. Per configuration, collecting `honestParticipants`
   543  			// number of votes is the minimally required supermajority.
   544  			require.GreaterOrEqual(t, uint64(len(blockSigData.StakingSigners)+len(blockSigData.RandomBeaconSigners)), honestParticipants)
   545  
   546  			// 3. CHECK: `AggregatedStakingSig`
   547  			// Here, we have to pay attention to the edge case where all replicas voted with their random beacon sig.
   548  			// Per protocol convention, the `AggregatedStakingSig` should be empty, for an empty set of StakingSigners.
   549  			if len(blockSigData.StakingSigners) == 0 {
   550  				require.Empty(t, blockSigData.AggregatedStakingSig)
   551  			} else {
   552  				// otherwise, we expect `AggregatedStakingSig` to be the return value of
   553  				// `stakingAggregator.Aggregate()`, which we mocked as `combinedSigs[0]`
   554  				require.Equal(t, []byte(combinedSigs[0]), blockSigData.AggregatedStakingSig)
   555  			}
   556  
   557  			// 4. CHECK: `AggregatedRandomBeaconSig` and `ReconstructedRandomBeaconSig`
   558  			// We require that each QC contains valid random beacon value, i.e. we must collect some votes with
   559  			// random beacon signatures to construct a valid QC. Hence, `AggregatedRandomBeaconSig` should be the
   560  			// output of `rbSigAggregator.Aggregate()`, which we mocked as `combinedSigs[1]`.
   561  			require.Equal(t, []byte(combinedSigs[1]), blockSigData.AggregatedRandomBeaconSig)
   562  			// Furthermore, `ReconstructedRandomBeaconSig` should be the output of `reconstructor.Reconstruct()`,
   563  			// which we mocked as `combinedSigs[2]`
   564  			require.Equal(t, combinedSigs[2], blockSigData.ReconstructedRandomBeaconSig)
   565  
   566  			// fill merged signers with collected signers
   567  			mergedSignerIDs = append(blockSigData.StakingSigners, blockSigData.RandomBeaconSigners...)
   568  		}).Return(
   569  			func(flow.Identifier, *hotstuff.BlockSignatureData) []byte {
   570  				signerIndices, _ := signature.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs)
   571  				return signerIndices
   572  			},
   573  			func(flow.Identifier, *hotstuff.BlockSignatureData) []byte { return packedSigData },
   574  			func(flow.Identifier, *hotstuff.BlockSignatureData) error { return nil }).Once()
   575  
   576  		// track if QC was created
   577  		qcCreated := atomic.NewBool(false)
   578  
   579  		// expected QC
   580  		onQCCreated := func(qc *flow.QuorumCertificate) {
   581  			// QC should be created only once
   582  			if !qcCreated.CompareAndSwap(false, true) {
   583  				t.Fatalf("QC created more than once")
   584  			}
   585  
   586  			signerIndices, err := signature.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs)
   587  			require.NoError(t, err)
   588  
   589  			// ensure that QC contains correct field
   590  			expectedQC := &flow.QuorumCertificate{
   591  				View:          block.View,
   592  				BlockID:       block.BlockID,
   593  				SignerIndices: signerIndices,
   594  				SigData:       packedSigData,
   595  			}
   596  			require.Equalf(t, expectedQC, qc, "QC should be equal to what we expect")
   597  		}
   598  
   599  		processor := &CombinedVoteProcessorV3{
   600  			log:               unittest.Logger(),
   601  			block:             block,
   602  			stakingSigAggtor:  stakingAggregator,
   603  			rbSigAggtor:       rbSigAggregator,
   604  			rbRector:          reconstructor,
   605  			onQCCreated:       onQCCreated,
   606  			packer:            pcker,
   607  			minRequiredWeight: minRequiredWeight,
   608  			done:              *atomic.NewBool(false),
   609  		}
   610  
   611  		votes := make([]*model.Vote, 0, stakingSignersCount+beaconSignersCount)
   612  
   613  		// prepare votes
   614  		for _, signer := range stakingSigners {
   615  			vote := unittest.VoteForBlockFixture(processor.Block(), unittest.VoteWithStakingSig())
   616  			vote.SignerID = signer
   617  			expectedSig := crypto.Signature(vote.SigData[1:])
   618  			stakingAggregator.On("Verify", vote.SignerID, expectedSig).Return(nil).Maybe()
   619  			stakingAggregator.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   620  				signerID := args.Get(0).(flow.Identifier)
   621  				stakingAggregatorLock.Lock()
   622  				defer stakingAggregatorLock.Unlock()
   623  				stakingTotalWeight += sigWeight
   624  				aggregatedStakingSigners = append(aggregatedStakingSigners, signerID)
   625  			}).Return(uint64(0), nil).Maybe()
   626  			votes = append(votes, vote)
   627  		}
   628  		for _, signer := range beaconSigners {
   629  			vote := unittest.VoteForBlockFixture(processor.Block(), unittest.VoteWithBeaconSig())
   630  			vote.SignerID = signer
   631  			expectedSig := crypto.Signature(vote.SigData[1:])
   632  			rbSigAggregator.On("Verify", vote.SignerID, expectedSig).Return(nil).Maybe()
   633  			rbSigAggregator.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   634  				signerID := args.Get(0).(flow.Identifier)
   635  				beaconAggregatorLock.Lock()
   636  				defer beaconAggregatorLock.Unlock()
   637  				thresholdTotalWeight += sigWeight
   638  				aggregatedBeaconSigners = append(aggregatedBeaconSigners, signerID)
   639  			}).Return(uint64(0), nil).Maybe()
   640  			reconstructor.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   641  				beaconReconstructorLock.Lock()
   642  				defer beaconReconstructorLock.Unlock()
   643  				collectedShares++
   644  			}).Return(true, nil).Maybe()
   645  			votes = append(votes, vote)
   646  		}
   647  
   648  		// shuffle votes in random order
   649  		rand.Seed(time.Now().UnixNano())
   650  		rand.Shuffle(len(votes), func(i, j int) {
   651  			votes[i], votes[j] = votes[j], votes[i]
   652  		})
   653  
   654  		var startProcessing, finishProcessing sync.WaitGroup
   655  		startProcessing.Add(1)
   656  		// process votes concurrently by multiple workers
   657  		for _, vote := range votes {
   658  			finishProcessing.Add(1)
   659  			go func(vote *model.Vote) {
   660  				defer finishProcessing.Done()
   661  				startProcessing.Wait()
   662  				err := processor.Process(vote)
   663  				require.NoError(t, err)
   664  			}(vote)
   665  		}
   666  
   667  		// start all goroutines at the same time
   668  		startProcessing.Done()
   669  		finishProcessing.Wait()
   670  
   671  		passed := processor.done.Load()
   672  		passed = passed && qcCreated.Load()
   673  		passed = passed && rbSigAggregator.AssertExpectations(t)
   674  		passed = passed && stakingAggregator.AssertExpectations(t)
   675  		passed = passed && reconstructor.AssertExpectations(t)
   676  
   677  		if !passed {
   678  			t.Fatalf("Assertions weren't met, staking weight: %v, threshold weight: %v", stakingTotalWeight, thresholdTotalWeight)
   679  		}
   680  
   681  		//processing extra votes shouldn't result in creating new QCs
   682  		vote := unittest.VoteForBlockFixture(block, unittest.VoteWithBeaconSig())
   683  		err := processor.Process(vote)
   684  		require.NoError(t, err)
   685  	})
   686  }
   687  
   688  // TestCombinedVoteProcessorV3_OnlyRandomBeaconSigners tests the most optimal happy path,
   689  // where all consensus replicas vote using their random beacon keys. In this case,
   690  // no staking signatures were collected and the CombinedVoteProcessor should be setting
   691  // `BlockSignatureData.StakingSigners` and `BlockSignatureData.AggregatedStakingSig` to nil or empty slices.
   692  func TestCombinedVoteProcessorV3_OnlyRandomBeaconSigners(testifyT *testing.T) {
   693  	// setup CombinedVoteProcessorV3
   694  	block := helper.MakeBlock()
   695  	stakingAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   696  	rbSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   697  	reconstructor := &mockhotstuff.RandomBeaconReconstructor{}
   698  	packer := &mockhotstuff.Packer{}
   699  
   700  	processor := &CombinedVoteProcessorV3{
   701  		log:               unittest.Logger(),
   702  		block:             block,
   703  		stakingSigAggtor:  stakingAggregator,
   704  		rbSigAggtor:       rbSigAggregator,
   705  		rbRector:          reconstructor,
   706  		onQCCreated:       func(qc *flow.QuorumCertificate) { /* no op */ },
   707  		packer:            packer,
   708  		minRequiredWeight: 70,
   709  		done:              *atomic.NewBool(false),
   710  	}
   711  
   712  	// The `stakingAggregator` is empty, i.e. it returns ans InsufficientSignaturesError when we call `Aggregate()` on it.
   713  	stakingAggregator.On("TotalWeight").Return(uint64(0), nil).Twice() // called a second time to determine whether there are any staking sigs to aggregate
   714  	stakingAggregator.On("Aggregate").Return(nil, nil, model.NewInsufficientSignaturesErrorf("")).Maybe()
   715  
   716  	// Create another vote with a random beacon signature. With its addition, the `rbSigAggregator`
   717  	// by itself has collected enough votes to exceed the minimally required weight (70).
   718  	vote := unittest.VoteForBlockFixture(block, unittest.VoteWithBeaconSig())
   719  	rawSig := (crypto.Signature)(vote.SigData[1:])
   720  	rbSigAggregator.On("Verify", vote.SignerID, rawSig).Return(nil).Once()
   721  	rbSigAggregator.On("TrustedAdd", vote.SignerID, rawSig).Return(uint64(80), nil).Once()
   722  	rbSigAggregator.On("TotalWeight").Return(uint64(80), nil).Once()
   723  	rbSigAggregator.On("Aggregate").Return(unittest.IdentifierListFixture(11), unittest.RandomBytes(48), nil).Once()
   724  	reconstructor.On("TrustedAdd", vote.SignerID, rawSig).Return(true, nil).Once()
   725  	reconstructor.On("EnoughShares").Return(true).Once()
   726  	reconstructor.On("Reconstruct").Return(unittest.SignatureFixture(), nil).Once()
   727  
   728  	// Adding the vote should trigger QC generation. We expect `BlockSignatureData.StakingSigners`
   729  	// and `BlockSignatureData.AggregatedStakingSig` to be both empty, as there are no staking signatures.
   730  	packer.On("Pack", block.BlockID, mock.Anything).
   731  		Run(func(args mock.Arguments) {
   732  			blockSigData := args.Get(1).(*hotstuff.BlockSignatureData)
   733  			require.Empty(testifyT, blockSigData.StakingSigners)
   734  			require.Empty(testifyT, blockSigData.AggregatedStakingSig)
   735  		}).
   736  		Return(unittest.RandomBytes(100), unittest.RandomBytes(1017), nil).Once()
   737  
   738  	err := processor.Process(vote)
   739  	require.NoError(testifyT, err)
   740  
   741  	stakingAggregator.AssertExpectations(testifyT)
   742  	rbSigAggregator.AssertExpectations(testifyT)
   743  	reconstructor.AssertExpectations(testifyT)
   744  	packer.AssertExpectations(testifyT)
   745  }
   746  
   747  // TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness uses property testing to test liveness of concurrent votes processing.
   748  // We randomly draw a committee and check if we are able to create a QC with minimal number of nodes.
   749  // In each test iteration we expect to create a QC, we don't check correctness of data since it's checked by another test.
   750  func TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness(testifyT *testing.T) {
   751  	rapid.Check(testifyT, func(t *rapid.T) {
   752  		// draw beacon signers in range 1 <= beaconSignersCount <= 53
   753  		beaconSignersCount := rapid.Uint64Range(1, 53).Draw(t, "beaconSigners").(uint64)
   754  		// draw staking signers in range 0 <= stakingSignersCount <= 10
   755  		stakingSignersCount := rapid.Uint64Range(0, 10).Draw(t, "stakingSigners").(uint64)
   756  
   757  		stakingWeightRange, beaconWeightRange := rapid.Uint64Range(1, 10), rapid.Uint64Range(1, 10)
   758  
   759  		minRequiredWeight := uint64(0)
   760  		// draw weight for each signer randomly
   761  		stakingSigners := unittest.IdentityListFixture(int(stakingSignersCount), func(identity *flow.Identity) {
   762  			identity.Weight = stakingWeightRange.Draw(t, identity.String()).(uint64)
   763  			minRequiredWeight += identity.Weight
   764  		})
   765  		beaconSigners := unittest.IdentityListFixture(int(beaconSignersCount), func(identity *flow.Identity) {
   766  			identity.Weight = beaconWeightRange.Draw(t, identity.String()).(uint64)
   767  			minRequiredWeight += identity.Weight
   768  		})
   769  
   770  		// proposing block
   771  		block := helper.MakeBlock()
   772  
   773  		t.Logf("running conf\n\t"+
   774  			"staking signers: %v, beacon signers: %v\n\t"+
   775  			"required weight: %v", stakingSignersCount, beaconSignersCount, minRequiredWeight)
   776  
   777  		stakingTotalWeight, thresholdTotalWeight, collectedShares := atomic.NewUint64(0), atomic.NewUint64(0), atomic.NewUint64(0)
   778  
   779  		// setup aggregators and reconstructor
   780  		stakingAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   781  		rbSigAggregator := &mockhotstuff.WeightedSignatureAggregator{}
   782  		reconstructor := &mockhotstuff.RandomBeaconReconstructor{}
   783  
   784  		stakingAggregator.On("TotalWeight").Return(func() uint64 {
   785  			return stakingTotalWeight.Load()
   786  		})
   787  		rbSigAggregator.On("TotalWeight").Return(func() uint64 {
   788  			return thresholdTotalWeight.Load()
   789  		})
   790  		// don't require shares
   791  		reconstructor.On("EnoughShares").Return(func() bool {
   792  			return collectedShares.Load() >= beaconSignersCount
   793  		})
   794  
   795  		// mock expected calls to aggregators and reconstructor
   796  		combinedSigs := unittest.SignaturesFixture(3)
   797  		stakingAggregator.On("Aggregate").Return(
   798  			// per API convention, model.InsufficientSignaturesError is returns when no signatures were collected
   799  			func() flow.IdentifierList {
   800  				if len(stakingSigners) == 0 {
   801  					return nil
   802  				}
   803  				return stakingSigners.NodeIDs()
   804  			},
   805  			func() []byte {
   806  				if len(stakingSigners) == 0 {
   807  					return nil
   808  				}
   809  				return combinedSigs[0]
   810  			},
   811  			func() error {
   812  				if len(stakingSigners) == 0 {
   813  					return model.NewInsufficientSignaturesErrorf("")
   814  				}
   815  				return nil
   816  			}).Maybe()
   817  		rbSigAggregator.On("Aggregate").Return(beaconSigners.NodeIDs(), []byte(combinedSigs[1]), nil).Once()
   818  		reconstructor.On("Reconstruct").Return(combinedSigs[2], nil).Once()
   819  
   820  		// mock expected call to Packer
   821  		mergedSignerIDs := append(stakingSigners.NodeIDs(), beaconSigners.NodeIDs()...)
   822  		packedSigData := unittest.RandomBytes(128)
   823  		pcker := &mockhotstuff.Packer{}
   824  
   825  		signerIndices, err := signature.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs)
   826  		require.NoError(t, err)
   827  		pcker.On("Pack", block.BlockID, mock.Anything).Return(signerIndices, packedSigData, nil)
   828  
   829  		// track if QC was created
   830  		qcCreated := atomic.NewBool(false)
   831  
   832  		// expected QC
   833  		onQCCreated := func(qc *flow.QuorumCertificate) {
   834  			// QC should be created only once
   835  			if !qcCreated.CompareAndSwap(false, true) {
   836  				t.Fatalf("QC created more than once")
   837  			}
   838  		}
   839  
   840  		processor := &CombinedVoteProcessorV3{
   841  			log:               unittest.Logger(),
   842  			block:             block,
   843  			stakingSigAggtor:  stakingAggregator,
   844  			rbSigAggtor:       rbSigAggregator,
   845  			rbRector:          reconstructor,
   846  			onQCCreated:       onQCCreated,
   847  			packer:            pcker,
   848  			minRequiredWeight: minRequiredWeight,
   849  			done:              *atomic.NewBool(false),
   850  		}
   851  
   852  		votes := make([]*model.Vote, 0, stakingSignersCount+beaconSignersCount)
   853  
   854  		// prepare votes
   855  		for _, signer := range stakingSigners {
   856  			vote := unittest.VoteForBlockFixture(processor.Block(), unittest.VoteWithStakingSig())
   857  			vote.SignerID = signer.ID()
   858  			weight := signer.Weight
   859  			expectedSig := crypto.Signature(vote.SigData[1:])
   860  			stakingAggregator.On("Verify", vote.SignerID, expectedSig).Return(nil).Maybe()
   861  			stakingAggregator.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   862  				stakingTotalWeight.Add(weight)
   863  			}).Return(uint64(0), nil).Maybe()
   864  			votes = append(votes, vote)
   865  		}
   866  		for _, signer := range beaconSigners {
   867  			vote := unittest.VoteForBlockFixture(processor.Block(), unittest.VoteWithBeaconSig())
   868  			vote.SignerID = signer.ID()
   869  			weight := signer.Weight
   870  			expectedSig := crypto.Signature(vote.SigData[1:])
   871  			rbSigAggregator.On("Verify", vote.SignerID, expectedSig).Return(nil).Maybe()
   872  			rbSigAggregator.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   873  				thresholdTotalWeight.Add(weight)
   874  			}).Return(uint64(0), nil).Maybe()
   875  			reconstructor.On("TrustedAdd", vote.SignerID, expectedSig).Run(func(args mock.Arguments) {
   876  				collectedShares.Inc()
   877  			}).Return(true, nil).Maybe()
   878  			votes = append(votes, vote)
   879  		}
   880  
   881  		// shuffle votes in random order
   882  		rand.Seed(time.Now().UnixNano())
   883  		rand.Shuffle(len(votes), func(i, j int) {
   884  			votes[i], votes[j] = votes[j], votes[i]
   885  		})
   886  
   887  		var startProcessing, finishProcessing sync.WaitGroup
   888  		startProcessing.Add(1)
   889  		// process votes concurrently by multiple workers
   890  		for _, vote := range votes {
   891  			finishProcessing.Add(1)
   892  			go func(vote *model.Vote) {
   893  				defer finishProcessing.Done()
   894  				startProcessing.Wait()
   895  				err := processor.Process(vote)
   896  				require.NoError(t, err)
   897  			}(vote)
   898  		}
   899  
   900  		// start all goroutines at the same time
   901  		startProcessing.Done()
   902  		finishProcessing.Wait()
   903  
   904  		passed := processor.done.Load()
   905  		passed = passed && qcCreated.Load()
   906  		passed = passed && rbSigAggregator.AssertExpectations(t)
   907  		passed = passed && stakingAggregator.AssertExpectations(t)
   908  		passed = passed && reconstructor.AssertExpectations(t)
   909  
   910  		if !passed {
   911  			t.Fatalf("Assertions weren't met, staking weight: %v, threshold weight: %v", stakingTotalWeight, thresholdTotalWeight)
   912  		}
   913  	})
   914  }
   915  
   916  // TestCombinedVoteProcessorV3_BuildVerifyQC tests a complete path from creating votes to collecting votes and then
   917  // building & verifying QC.
   918  // We start with leader proposing a block, then new leader collects votes and builds a QC.
   919  // Need to verify that QC that was produced is valid and can be embedded in new proposal.
   920  func TestCombinedVoteProcessorV3_BuildVerifyQC(t *testing.T) {
   921  	epochCounter := uint64(3)
   922  	epochLookup := &modulemock.EpochLookup{}
   923  	view := uint64(20)
   924  	epochLookup.On("EpochForViewWithFallback", view).Return(epochCounter, nil)
   925  
   926  	dkgData, err := bootstrapDKG.RunFastKG(11, unittest.RandomBytes(32))
   927  	require.NoError(t, err)
   928  
   929  	// signers hold objects that are created with private key and can sign votes and proposals
   930  	signers := make(map[flow.Identifier]*verification.CombinedSignerV3)
   931  
   932  	// prepare staking signers, each signer has it's own private/public key pair
   933  	// stakingSigners sign only with staking key, meaning they have failed DKG
   934  	stakingSigners := unittest.IdentityListFixture(3)
   935  	beaconSigners := unittest.IdentityListFixture(8)
   936  	allIdentities := append(stakingSigners, beaconSigners...)
   937  	require.Equal(t, len(dkgData.PubKeyShares), len(allIdentities))
   938  	dkgParticipants := make(map[flow.Identifier]flow.DKGParticipant)
   939  	// fill dkg participants data
   940  	for index, identity := range allIdentities {
   941  		dkgParticipants[identity.NodeID] = flow.DKGParticipant{
   942  			Index:    uint(index),
   943  			KeyShare: dkgData.PubKeyShares[index],
   944  		}
   945  	}
   946  
   947  	for _, identity := range stakingSigners {
   948  		stakingPriv := unittest.StakingPrivKeyFixture()
   949  		identity.StakingPubKey = stakingPriv.PublicKey()
   950  
   951  		keys := &storagemock.SafeBeaconKeys{}
   952  		// there is no DKG key for this epoch
   953  		keys.On("RetrieveMyBeaconPrivateKey", epochCounter).Return(nil, false, nil)
   954  
   955  		beaconSignerStore := hsig.NewEpochAwareRandomBeaconKeyStore(epochLookup, keys)
   956  
   957  		me, err := local.New(identity, stakingPriv)
   958  		require.NoError(t, err)
   959  
   960  		signers[identity.NodeID] = verification.NewCombinedSignerV3(me, beaconSignerStore)
   961  	}
   962  
   963  	for _, identity := range beaconSigners {
   964  		stakingPriv := unittest.StakingPrivKeyFixture()
   965  		identity.StakingPubKey = stakingPriv.PublicKey()
   966  
   967  		participantData := dkgParticipants[identity.NodeID]
   968  
   969  		dkgKey := encodable.RandomBeaconPrivKey{
   970  			PrivateKey: dkgData.PrivKeyShares[participantData.Index],
   971  		}
   972  
   973  		keys := &storagemock.SafeBeaconKeys{}
   974  		// there is DKG key for this epoch
   975  		keys.On("RetrieveMyBeaconPrivateKey", epochCounter).Return(dkgKey, true, nil)
   976  
   977  		beaconSignerStore := hsig.NewEpochAwareRandomBeaconKeyStore(epochLookup, keys)
   978  
   979  		me, err := local.New(identity, stakingPriv)
   980  		require.NoError(t, err)
   981  
   982  		signers[identity.NodeID] = verification.NewCombinedSignerV3(me, beaconSignerStore)
   983  	}
   984  
   985  	leader := stakingSigners[0]
   986  
   987  	block := helper.MakeBlock(helper.WithBlockView(view),
   988  		helper.WithBlockProposer(leader.NodeID))
   989  
   990  	inmemDKG, err := inmem.DKGFromEncodable(inmem.EncodableDKG{
   991  		GroupKey: encodable.RandomBeaconPubKey{
   992  			PublicKey: dkgData.PubGroupKey,
   993  		},
   994  		Participants: dkgParticipants,
   995  	})
   996  	require.NoError(t, err)
   997  
   998  	committee := &mockhotstuff.Committee{}
   999  	committee.On("Identities", block.BlockID, mock.Anything).Return(allIdentities, nil)
  1000  	committee.On("DKG", block.BlockID).Return(inmemDKG, nil)
  1001  
  1002  	votes := make([]*model.Vote, 0, len(allIdentities))
  1003  
  1004  	// first staking signer will be leader collecting votes for proposal
  1005  	// prepare votes for every member of committee except leader
  1006  	for _, signer := range allIdentities[1:] {
  1007  		vote, err := signers[signer.NodeID].CreateVote(block)
  1008  		require.NoError(t, err)
  1009  		votes = append(votes, vote)
  1010  	}
  1011  
  1012  	// create and sign proposal
  1013  	proposal, err := signers[leader.NodeID].CreateProposal(block)
  1014  	require.NoError(t, err)
  1015  
  1016  	qcCreated := false
  1017  	onQCCreated := func(qc *flow.QuorumCertificate) {
  1018  		packer := hsig.NewConsensusSigDataPacker(committee)
  1019  
  1020  		// create verifier that will do crypto checks of created QC
  1021  		verifier := verification.NewCombinedVerifierV3(committee, packer)
  1022  		forks := &mockhotstuff.Forks{}
  1023  		// create validator which will do compliance and crypto checked of created QC
  1024  		validator := hotstuffvalidator.New(committee, forks, verifier)
  1025  		// check if QC is valid against parent
  1026  		err := validator.ValidateQC(qc, block)
  1027  		require.NoError(t, err)
  1028  
  1029  		qcCreated = true
  1030  	}
  1031  
  1032  	baseFactory := &combinedVoteProcessorFactoryBaseV3{
  1033  		committee:   committee,
  1034  		onQCCreated: onQCCreated,
  1035  		packer:      hsig.NewConsensusSigDataPacker(committee),
  1036  	}
  1037  	voteProcessorFactory := &VoteProcessorFactory{
  1038  		baseFactory: baseFactory.Create,
  1039  	}
  1040  	voteProcessor, err := voteProcessorFactory.Create(unittest.Logger(), proposal)
  1041  	require.NoError(t, err)
  1042  
  1043  	// process votes by new leader, this will result in producing new QC
  1044  	for _, vote := range votes {
  1045  		err := voteProcessor.Process(vote)
  1046  		require.NoError(t, err)
  1047  	}
  1048  
  1049  	require.True(t, qcCreated)
  1050  }