github.com/Blockdaemon/celo-blockchain@v0.0.0-20200129231733-e667f6b08419/consensus/istanbul/core/roundchange_test.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package core
    18  
    19  import (
    20  	"math/big"
    21  	"testing"
    22  	"time"
    23  
    24  	"github.com/ethereum/go-ethereum/common"
    25  	"github.com/ethereum/go-ethereum/consensus/istanbul"
    26  	"github.com/ethereum/go-ethereum/consensus/istanbul/validator"
    27  )
    28  
    29  func TestRoundChangeSet(t *testing.T) {
    30  	vals, _, _ := generateValidators(4)
    31  	vset := validator.NewSet(vals)
    32  	rc := newRoundChangeSet(vset)
    33  
    34  	view := &istanbul.View{
    35  		Sequence: big.NewInt(1),
    36  		Round:    big.NewInt(1),
    37  	}
    38  	r := &istanbul.Subject{
    39  		View:   view,
    40  		Digest: common.Hash{},
    41  	}
    42  	m, _ := Encode(r)
    43  
    44  	// Test Add()
    45  	// Add message from all validators
    46  	for i, v := range vset.List() {
    47  		msg := &istanbul.Message{
    48  			Code:    istanbul.MsgRoundChange,
    49  			Msg:     m,
    50  			Address: v.Address(),
    51  		}
    52  		rc.Add(view.Round, msg)
    53  		if rc.msgsForRound[view.Round.Uint64()].Size() != i+1 {
    54  			t.Errorf("the size of round change messages mismatch: have %v, want %v", rc.msgsForRound[view.Round.Uint64()].Size(), i+1)
    55  		}
    56  	}
    57  
    58  	// Add message again from all validators, but the size should be the same
    59  	for _, v := range vset.List() {
    60  		msg := &istanbul.Message{
    61  			Code:    istanbul.MsgRoundChange,
    62  			Msg:     m,
    63  			Address: v.Address(),
    64  		}
    65  		rc.Add(view.Round, msg)
    66  		if rc.msgsForRound[view.Round.Uint64()].Size() != vset.Size() {
    67  			t.Errorf("the size of round change messages mismatch: have %v, want %v", rc.msgsForRound[view.Round.Uint64()].Size(), vset.Size())
    68  		}
    69  	}
    70  
    71  	// Test MaxRound()
    72  	for i := 0; i < 10; i++ {
    73  		maxRound := rc.MaxRound(i)
    74  		if i <= vset.Size() {
    75  			if maxRound == nil || maxRound.Cmp(view.Round) != 0 {
    76  				t.Errorf("MaxRound mismatch: have %v, want %v", maxRound, view.Round)
    77  			}
    78  		} else if maxRound != nil {
    79  			t.Errorf("MaxRound mismatch: have %v, want nil", maxRound)
    80  		}
    81  	}
    82  
    83  	// Test Clear()
    84  	for i := int64(0); i < 2; i++ {
    85  		rc.Clear(big.NewInt(i))
    86  		if rc.msgsForRound[view.Round.Uint64()].Size() != vset.Size() {
    87  			t.Errorf("the size of round change messages mismatch: have %v, want %v", rc.msgsForRound[view.Round.Uint64()].Size(), vset.Size())
    88  		}
    89  	}
    90  	rc.Clear(big.NewInt(2))
    91  	if rc.msgsForRound[view.Round.Uint64()] != nil {
    92  		t.Errorf("the change messages mismatch: have %v, want nil", rc.msgsForRound[view.Round.Uint64()])
    93  	}
    94  
    95  	// Test Add()
    96  	// Add message from all validators
    97  	for i, v := range vset.List() {
    98  		msg := &istanbul.Message{
    99  			Code:    istanbul.MsgRoundChange,
   100  			Msg:     m,
   101  			Address: v.Address(),
   102  		}
   103  		rc.Add(view.Round, msg)
   104  		if rc.msgsForRound[view.Round.Uint64()].Size() != i+1 {
   105  			t.Errorf("the size of round change messages mismatch: have %v, want %v", rc.msgsForRound[view.Round.Uint64()].Size(), i+1)
   106  		}
   107  	}
   108  
   109  	rc.Clear(big.NewInt(2))
   110  	if rc.msgsForRound[view.Round.Uint64()] != nil {
   111  		t.Errorf("the change messages mismatch: have %v, want nil", rc.msgsForRound[view.Round.Uint64()])
   112  	}
   113  
   114  	// Test that we only store the msg with the highest round for each validator
   115  	roundMultiplier := 1
   116  	for j := 1; j <= roundMultiplier; j++ {
   117  		for i, v := range vset.List() {
   118  			view := &istanbul.View{
   119  				Sequence: big.NewInt(1),
   120  				Round:    big.NewInt(int64((i + 1) * j)),
   121  			}
   122  			r := &istanbul.Subject{
   123  				View:   view,
   124  				Digest: common.Hash{},
   125  			}
   126  			m, _ := Encode(r)
   127  			msg := &istanbul.Message{
   128  				Code:    istanbul.MsgRoundChange,
   129  				Msg:     m,
   130  				Address: v.Address(),
   131  			}
   132  			err := rc.Add(view.Round, msg)
   133  			if err != nil {
   134  				t.Errorf("Round change message: unexpected error %v", err)
   135  			}
   136  		}
   137  	}
   138  
   139  	for i, v := range vset.List() {
   140  		lookingForValAtRound := uint64(roundMultiplier * (i + 1))
   141  		if rc.msgsForRound[lookingForValAtRound].Size() != 1 {
   142  			t.Errorf("Round change messages at unexpected rounds: %v", rc.msgsForRound)
   143  		}
   144  		if rc.latestRoundForVal[v.Address()] != lookingForValAtRound {
   145  			t.Errorf("Round change messages at unexpected rounds: for %v want %v have %v",
   146  				i+1, rc.latestRoundForVal[v.Address()], lookingForValAtRound)
   147  		}
   148  	}
   149  
   150  	for threshold := 1; threshold <= vset.Size(); threshold++ {
   151  		r := rc.MaxRound(threshold).Uint64()
   152  		expectedR := uint64((vset.Size() - threshold + 1) * roundMultiplier)
   153  		if r != expectedR {
   154  			t.Errorf("MaxRound: %v want %v have %v", rc.String(), expectedR, r)
   155  		}
   156  	}
   157  
   158  	// Test getCertificate
   159  	for r := 1; r < vset.Size(); r += roundMultiplier {
   160  		expectedMsgsAtRound := vset.Size() - r + 1
   161  		for quorum := 1; quorum < 10; quorum++ {
   162  			cert, err := rc.getCertificate(big.NewInt(int64(r)), quorum)
   163  			if expectedMsgsAtRound < quorum {
   164  				// Expecting fewer than quorum.
   165  				if err != errFailedCreateRoundChangeCertificate || len(cert.RoundChangeMessages) != 0 {
   166  					t.Errorf("problem in getCertificate r=%v q=%v expMsgs=%v - want 0 have %v err=%v -- %v -- %v", r, quorum, expectedMsgsAtRound, len(cert.RoundChangeMessages), err, cert, rc)
   167  				}
   168  			} else {
   169  				// Number msgs available at this round is >= quorum. Expecting a cert with =quorum RC messages.
   170  				if err != nil || len(cert.RoundChangeMessages) != quorum {
   171  					t.Errorf("problem in getCertificate r=%v q=%v expMsgs=%v - want %v have %v -- %v -- %v", r, quorum, quorum, expectedMsgsAtRound, len(cert.RoundChangeMessages), cert, rc)
   172  				}
   173  			}
   174  		}
   175  	}
   176  }
   177  
   178  func TestHandleRoundChangeCertificate(t *testing.T) {
   179  	N := uint64(4) // replica 0 is the proposer, it will send messages to others
   180  	F := uint64(1)
   181  	view := istanbul.View{
   182  		Round:    big.NewInt(1),
   183  		Sequence: big.NewInt(1),
   184  	}
   185  
   186  	testCases := []struct {
   187  		name           string
   188  		getCertificate func(*testing.T, *testSystem) istanbul.RoundChangeCertificate
   189  		expectedErr    error
   190  	}{
   191  		{
   192  			"Valid round change certificate without PREPARED certificate",
   193  			func(t *testing.T, sys *testSystem) istanbul.RoundChangeCertificate {
   194  				return sys.getRoundChangeCertificate(t, []istanbul.View{view}, istanbul.EmptyPreparedCertificate())
   195  			},
   196  			nil,
   197  		},
   198  		{
   199  			"Valid round change certificate with PREPARED certificate",
   200  			func(t *testing.T, sys *testSystem) istanbul.RoundChangeCertificate {
   201  				return sys.getRoundChangeCertificate(t, []istanbul.View{view}, sys.getPreparedCertificate(t, []istanbul.View{view}, makeBlock(0)))
   202  			},
   203  			nil,
   204  		},
   205  		{
   206  			"Invalid round change certificate, duplicate message",
   207  			func(t *testing.T, sys *testSystem) istanbul.RoundChangeCertificate {
   208  				roundChangeCertificate := sys.getRoundChangeCertificate(t, []istanbul.View{view}, istanbul.EmptyPreparedCertificate())
   209  				roundChangeCertificate.RoundChangeMessages[1] = roundChangeCertificate.RoundChangeMessages[0]
   210  				return roundChangeCertificate
   211  			},
   212  			errInvalidRoundChangeCertificateDuplicate,
   213  		},
   214  		{
   215  			"Empty certificate",
   216  			func(t *testing.T, sys *testSystem) istanbul.RoundChangeCertificate {
   217  				return istanbul.RoundChangeCertificate{}
   218  			},
   219  			errInvalidRoundChangeCertificateNumMsgs,
   220  		},
   221  	}
   222  	for _, test := range testCases {
   223  		t.Run(test.name, func(t *testing.T) {
   224  			sys := NewTestSystemWithBackend(N, F)
   225  			for i, backend := range sys.backends {
   226  				c := backend.engine.(*core)
   227  				c.Start()
   228  				certificate := test.getCertificate(t, sys)
   229  				subject := istanbul.Subject{
   230  					View:   &view,
   231  					Digest: makeBlock(0).Hash(),
   232  				}
   233  				err := c.handleRoundChangeCertificate(subject, certificate)
   234  
   235  				if err != test.expectedErr {
   236  					t.Errorf("error mismatch for test case %v: have %v, want %v", i, err, test.expectedErr)
   237  				}
   238  				if err == nil && c.current.View().Cmp(&view) != 0 {
   239  					t.Errorf("view mismatch for test case %v: have %v, want %v", i, c.current.View(), view)
   240  				}
   241  			}
   242  
   243  		})
   244  	}
   245  }
   246  
   247  func TestHandleRoundChange(t *testing.T) {
   248  	N := uint64(4) // replica 0 is the proposer, it will send messages to others
   249  	F := uint64(1) // F does not affect tests
   250  
   251  	buildEmptyCertificate := func(_ *testing.T, _ *testSystem) istanbul.PreparedCertificate {
   252  		return istanbul.EmptyPreparedCertificate()
   253  	}
   254  
   255  	noopPrepare := func(_ *testSystem) {}
   256  
   257  	testCases := []struct {
   258  		name          string
   259  		prepareSystem func(*testSystem)
   260  		getCert       func(*testing.T, *testSystem) istanbul.PreparedCertificate
   261  		expectedErr   error
   262  	}{
   263  		{
   264  			"normal case",
   265  			noopPrepare,
   266  			buildEmptyCertificate,
   267  			nil,
   268  		},
   269  		{
   270  			"normal case with valid prepared certificate",
   271  			noopPrepare,
   272  			func(t *testing.T, sys *testSystem) istanbul.PreparedCertificate {
   273  				return sys.getPreparedCertificate(t, []istanbul.View{*sys.backends[0].engine.(*core).current.View()}, makeBlock(1))
   274  			},
   275  			nil,
   276  		},
   277  		{
   278  			"normal case with invalid prepared certificate",
   279  			noopPrepare,
   280  			func(t *testing.T, sys *testSystem) istanbul.PreparedCertificate {
   281  				preparedCert := sys.getPreparedCertificate(t, []istanbul.View{*sys.backends[0].engine.(*core).current.View()}, makeBlock(1))
   282  				preparedCert.PrepareOrCommitMessages[0] = preparedCert.PrepareOrCommitMessages[1]
   283  				return preparedCert
   284  			},
   285  			errInvalidPreparedCertificateDuplicate,
   286  		},
   287  		{
   288  			"valid message for future round",
   289  			func(sys *testSystem) {
   290  				sys.backends[0].engine.(*core).current.(*rsSaveDecorator).rs.(*roundStateImpl).round = big.NewInt(10)
   291  			},
   292  			func(t *testing.T, _ *testSystem) istanbul.PreparedCertificate {
   293  				return istanbul.EmptyPreparedCertificate()
   294  			},
   295  			nil,
   296  		},
   297  		{
   298  			"invalid message for future sequence",
   299  			func(sys *testSystem) {
   300  				sys.backends[0].engine.(*core).current.(*rsSaveDecorator).rs.(*roundStateImpl).sequence = big.NewInt(10)
   301  			},
   302  			buildEmptyCertificate,
   303  			errFutureMessage,
   304  		},
   305  		{
   306  			"invalid message for previous round",
   307  			func(sys *testSystem) {
   308  				sys.backends[0].engine.(*core).current.(*rsSaveDecorator).rs.(*roundStateImpl).round = big.NewInt(0)
   309  			},
   310  			buildEmptyCertificate,
   311  			nil,
   312  		},
   313  	}
   314  
   315  	for _, test := range testCases {
   316  		t.Run(test.name, func(t *testing.T) {
   317  			sys := NewTestSystemWithBackend(N, F)
   318  
   319  			closer := sys.Run(false)
   320  			for _, v := range sys.backends {
   321  				v.engine.(*core).Start()
   322  			}
   323  			test.prepareSystem(sys)
   324  
   325  			v0 := sys.backends[0]
   326  			r0 := v0.engine.(*core)
   327  
   328  			curView := r0.current.View()
   329  			nextView := &istanbul.View{
   330  				Round:    new(big.Int).Add(curView.Round, common.Big1),
   331  				Sequence: curView.Sequence,
   332  			}
   333  
   334  			roundChange := &istanbul.RoundChange{
   335  				View:                nextView,
   336  				PreparedCertificate: test.getCert(t, sys),
   337  			}
   338  
   339  			for i, v := range sys.backends {
   340  				// i == 0 is primary backend, it is responsible for send ROUND CHANGE messages to others.
   341  				if i == 0 {
   342  					continue
   343  				}
   344  
   345  				c := v.engine.(*core)
   346  
   347  				m, _ := Encode(roundChange)
   348  
   349  				// run each backends and verify handlePreprepare function.
   350  				err := c.handleRoundChange(&istanbul.Message{
   351  					Code:    istanbul.MsgRoundChange,
   352  					Msg:     m,
   353  					Address: v0.Address(),
   354  				})
   355  				if err != test.expectedErr {
   356  					t.Errorf("error mismatch: have %v, want %v", err, test.expectedErr)
   357  				}
   358  				return
   359  			}
   360  
   361  			closer()
   362  		})
   363  	}
   364  }
   365  
   366  func (ts *testSystem) distributeIstMsgs(t *testing.T, sys *testSystem, istMsgDistribution map[uint64]map[int]bool) {
   367  	for {
   368  		select {
   369  		case <-ts.quit:
   370  			return
   371  		case event := <-ts.queuedMessage:
   372  			msg := new(istanbul.Message)
   373  			if err := msg.FromPayload(event.Payload, nil); err != nil {
   374  				t.Errorf("Could not decode payload")
   375  			}
   376  
   377  			targets := istMsgDistribution[msg.Code]
   378  			for index, b := range sys.backends {
   379  				if targets[index] || msg.Address == b.address {
   380  					go b.EventMux().Post(event)
   381  				} else {
   382  					testLogger.Info("ignoring message with code", "code", msg.Code)
   383  				}
   384  			}
   385  		}
   386  	}
   387  }
   388  
   389  var gossip = map[int]bool{
   390  	0: true,
   391  	1: true,
   392  	2: true,
   393  	3: true,
   394  }
   395  
   396  var sendTo2FPlus1 = map[int]bool{
   397  	0: true,
   398  	1: true,
   399  	2: true,
   400  	3: false,
   401  }
   402  
   403  var sendToF = map[int]bool{
   404  	0: false,
   405  	1: false,
   406  	2: false,
   407  	3: true,
   408  }
   409  
   410  var sendToFPlus1 = map[int]bool{
   411  	0: false,
   412  	1: false,
   413  	2: true,
   414  	3: true,
   415  }
   416  var noGossip = map[int]bool{
   417  	0: false,
   418  	1: false,
   419  	2: false,
   420  	3: false,
   421  }
   422  
   423  // This tests the liveness issue present in the initial implementation of Istanbul, described in
   424  // more detail here: https://arxiv.org/pdf/1901.07160.pdf
   425  // To test this, a block is proposed, for which 2F + 1 PREPARE messages are sent to F nodes.
   426  // In the original implementation, these F nodes would lock onto that block, and eventually everyone would
   427  // round change. If the next proposer was byzantine, they could send a PREPREPARE with a different block,
   428  // get the remaining 2F non-byzantine nodes to lock onto that new block, causing a deadlock.
   429  // In the new implementation, the PREPREPARE will include a ROUND CHANGE certificate,
   430  // and all nodes will accept the newly proposed block.
   431  func TestCommitsBlocksAfterRoundChange(t *testing.T) {
   432  	sys := NewTestSystemWithBackend(4, 1)
   433  
   434  	for i, b := range sys.backends {
   435  		b.engine.Start() // start Istanbul core
   436  		block := makeBlockWithDifficulty(1, int64(i))
   437  		b.NewRequest(block)
   438  	}
   439  
   440  	newBlocks := sys.backends[3].EventMux().Subscribe(istanbul.FinalCommittedEvent{})
   441  	defer newBlocks.Unsubscribe()
   442  
   443  	timeout := sys.backends[3].EventMux().Subscribe(timeoutAndMoveToNextRoundEvent{})
   444  	defer timeout.Unsubscribe()
   445  
   446  	istMsgDistribution := map[uint64]map[int]bool{}
   447  
   448  	// Allow everyone to see the initial proposal
   449  	// Send all PREPARE messages to F nodes.
   450  	// Send COMMIT messages (we don't expect these to be sent in the first round anyway).
   451  	// Send ROUND CHANGE messages to the remaining 2F + 1 nodes.
   452  	istMsgDistribution[istanbul.MsgPreprepare] = gossip
   453  	istMsgDistribution[istanbul.MsgPrepare] = sendToF
   454  	istMsgDistribution[istanbul.MsgCommit] = gossip
   455  	istMsgDistribution[istanbul.MsgRoundChange] = sendTo2FPlus1
   456  
   457  	go sys.distributeIstMsgs(t, sys, istMsgDistribution)
   458  
   459  	// Turn PREPAREs back on for round 1.
   460  	<-time.After(1 * time.Second)
   461  	istMsgDistribution[istanbul.MsgPrepare] = gossip
   462  
   463  	// Wait for round 1 to start.
   464  	<-timeout.Chan()
   465  
   466  	// Eventually we should get a block again
   467  	select {
   468  	case <-timeout.Chan():
   469  		t.Error("Did not finalize a block in round 1")
   470  	case _, ok := <-newBlocks.Chan():
   471  		if !ok {
   472  			t.Error("Error reading block")
   473  		}
   474  		// Wait for all backends to finalize the block.
   475  		<-time.After(1 * time.Second)
   476  		expectedCommitted, _ := sys.backends[0].GetCurrentHeadBlockAndAuthor()
   477  		for i, b := range sys.backends {
   478  			committed, _ := b.GetCurrentHeadBlockAndAuthor()
   479  			// We don't expect any particular block to be committed here. We do expect them to be consistent.
   480  			if committed.Number().Cmp(common.Big1) != 0 {
   481  				t.Errorf("Backend %v got committed block with unexpected number: expected %v, got %v", i, 1, committed.Number())
   482  			}
   483  			if expectedCommitted.Hash() != committed.Hash() {
   484  				t.Errorf("Backend %v got committed block with unexpected hash: expected %v, got %v", i, expectedCommitted.Hash(), committed.Hash())
   485  			}
   486  		}
   487  	}
   488  
   489  	// Manually open and close b/c hijacking sys.listen
   490  	for _, b := range sys.backends {
   491  		b.engine.Stop() // stop Istanbul core
   492  	}
   493  	close(sys.quit)
   494  }
   495  
   496  // This tests that when F+1 nodes receive 2F+1 PREPARE messages for a particular proposal, the
   497  // system enforces that as the only valid proposal for this sequence.
   498  func TestPreparedCertificatePersistsThroughRoundChanges(t *testing.T) {
   499  	sys := NewTestSystemWithBackend(4, 1)
   500  
   501  	for i, b := range sys.backends {
   502  		b.engine.Start() // start Istanbul core
   503  		block := makeBlockWithDifficulty(1, int64(i))
   504  		b.NewRequest(block)
   505  	}
   506  
   507  	newBlocks := sys.backends[3].EventMux().Subscribe(istanbul.FinalCommittedEvent{})
   508  	defer newBlocks.Unsubscribe()
   509  
   510  	timeout := sys.backends[3].EventMux().Subscribe(timeoutAndMoveToNextRoundEvent{})
   511  	defer timeout.Unsubscribe()
   512  
   513  	istMsgDistribution := map[uint64]map[int]bool{}
   514  
   515  	// Send PREPARE messages to F + 1 nodes so we guarantee a PREPARED certificate in the ROUND CHANGE certificate..
   516  	istMsgDistribution[istanbul.MsgPreprepare] = gossip
   517  	istMsgDistribution[istanbul.MsgPrepare] = sendToFPlus1
   518  	istMsgDistribution[istanbul.MsgCommit] = gossip
   519  	istMsgDistribution[istanbul.MsgRoundChange] = gossip
   520  
   521  	go sys.distributeIstMsgs(t, sys, istMsgDistribution)
   522  
   523  	// Turn PREPARE messages off for round 1 to force reuse of the PREPARED certificate.
   524  	<-time.After(1 * time.Second)
   525  	istMsgDistribution[istanbul.MsgPrepare] = noGossip
   526  
   527  	// Wait for round 1 to start.
   528  	<-timeout.Chan()
   529  	// Turn PREPARE messages back on in time for round 2.
   530  	<-time.After(1 * time.Second)
   531  	istMsgDistribution[istanbul.MsgPrepare] = gossip
   532  
   533  	// Wait for round 2 to start.
   534  	<-timeout.Chan()
   535  
   536  	select {
   537  	case <-timeout.Chan():
   538  		t.Error("Did not finalize a block in round 2.")
   539  	case _, ok := <-newBlocks.Chan():
   540  		if !ok {
   541  			t.Error("Error reading block")
   542  		}
   543  		// Wait for all backends to finalize the block.
   544  		<-time.After(2 * time.Second)
   545  		for i, b := range sys.backends {
   546  			committed, _ := b.GetCurrentHeadBlockAndAuthor()
   547  			// We expect to commit the block proposed by the first proposer.
   548  			expectedCommitted := makeBlockWithDifficulty(1, 0)
   549  			if committed.Number().Cmp(common.Big1) != 0 {
   550  				t.Errorf("Backend %v got committed block with unexpected number: expected %v, got %v", i, 1, committed.Number())
   551  			}
   552  			if expectedCommitted.Hash() != committed.Hash() {
   553  				t.Errorf("Backend %v got committed block with unexpected hash: expected %v, got %v", i, expectedCommitted.Hash(), committed.Hash())
   554  			}
   555  		}
   556  	}
   557  
   558  	// Manually open and close b/c hijacking sys.listen
   559  	for _, b := range sys.backends {
   560  		b.engine.Stop() // stop Istanbul core
   561  	}
   562  	close(sys.quit)
   563  }
   564  
   565  // Test periodic round changes at high rounds
   566  func TestPeriodicRoundChanges(t *testing.T) {
   567  	sys := NewTestSystemWithBackend(4, 1)
   568  
   569  	for i, b := range sys.backends {
   570  		b.engine.Start() // start Istanbul core
   571  		block := makeBlockWithDifficulty(1, int64(i))
   572  		b.NewRequest(block)
   573  	}
   574  
   575  	newBlocks := sys.backends[3].EventMux().Subscribe(istanbul.FinalCommittedEvent{})
   576  	defer newBlocks.Unsubscribe()
   577  
   578  	timeoutMoveToNextRound := sys.backends[3].EventMux().Subscribe(timeoutAndMoveToNextRoundEvent{})
   579  	defer timeoutMoveToNextRound.Unsubscribe()
   580  
   581  	timeoutResendRC := sys.backends[3].EventMux().Subscribe(resendRoundChangeEvent{})
   582  	defer timeoutResendRC.Unsubscribe()
   583  
   584  	istMsgDistribution := map[uint64]map[int]bool{}
   585  	istMsgDistribution[istanbul.MsgPreprepare] = noGossip
   586  	istMsgDistribution[istanbul.MsgPrepare] = noGossip
   587  	istMsgDistribution[istanbul.MsgCommit] = noGossip
   588  	istMsgDistribution[istanbul.MsgRoundChange] = noGossip
   589  
   590  	go sys.distributeIstMsgs(t, sys, istMsgDistribution)
   591  
   592  	for _, b := range sys.backends {
   593  		b.engine.(*core).waitForDesiredRound(big.NewInt(5))
   594  	}
   595  
   596  	// Expect at least one repeat RC before move to next round.
   597  	timeoutResends := 0
   598  loop:
   599  	for {
   600  		select {
   601  		case <-timeoutResendRC.Chan():
   602  			testLogger.Info("Got timeoutResendRC")
   603  			timeoutResends++
   604  		case <-timeoutMoveToNextRound.Chan():
   605  			if timeoutResends == 0 {
   606  				t.Errorf("No Repeat events before moving to next round")
   607  			}
   608  			break loop
   609  		}
   610  	}
   611  
   612  	istMsgDistribution[istanbul.MsgPreprepare] = gossip
   613  	istMsgDistribution[istanbul.MsgPrepare] = gossip
   614  	istMsgDistribution[istanbul.MsgCommit] = gossip
   615  	istMsgDistribution[istanbul.MsgRoundChange] = gossip
   616  
   617  	// Make sure we finalize block in next round.
   618  	select {
   619  	case <-timeoutMoveToNextRound.Chan():
   620  		t.Error("Did not finalize a block.")
   621  	case _, ok := <-newBlocks.Chan():
   622  		if !ok {
   623  			t.Error("Error reading block")
   624  		}
   625  		// Wait for all backends to finalize the block.
   626  		<-time.After(2 * time.Second)
   627  		for i, b := range sys.backends {
   628  			committed, _ := b.GetCurrentHeadBlockAndAuthor()
   629  			// We expect to commit the block proposed by proposer 6 mod 4 = 2.
   630  			expectedCommitted := makeBlockWithDifficulty(1, 2)
   631  			if committed.Number().Cmp(common.Big1) != 0 {
   632  				t.Errorf("Backend %v got committed block with unexpected number: expected %v, got %v", i, 1, committed.Number())
   633  			}
   634  			if expectedCommitted.Hash() != committed.Hash() {
   635  				t.Errorf("Backend %v got committed block with unexpected hash: expected %v, got %v", i, expectedCommitted.Hash(), committed.Hash())
   636  			}
   637  		}
   638  	}
   639  
   640  	// Manually open and close b/c hijacking sys.listen
   641  	for _, b := range sys.backends {
   642  		b.engine.Stop() // stop Istanbul core
   643  	}
   644  	close(sys.quit)
   645  }