github.com/leonlxy/hyperledger@v1.0.0-alpha.0.20170427033203-34922035d248/orderer/network_test.go (about)

     1  /*
     2  Copyright Digital Asset Holdings, LLC 2016 All Rights Reserved.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"crypto/rand"
    21  	"crypto/x509"
    22  	"crypto/x509/pkix"
    23  	"encoding/pem"
    24  	"fmt"
    25  	"io/ioutil"
    26  	"math"
    27  	"math/big"
    28  	"os"
    29  	"os/exec"
    30  	"runtime"
    31  	"testing"
    32  	"time"
    33  
    34  	"encoding/json"
    35  
    36  	"github.com/golang/protobuf/proto"
    37  	genesisconfig "github.com/hyperledger/fabric/common/configtx/tool/localconfig"
    38  	"github.com/hyperledger/fabric/common/configtx/tool/provisional"
    39  	cb "github.com/hyperledger/fabric/protos/common"
    40  	ab "github.com/hyperledger/fabric/protos/orderer"
    41  	"github.com/hyperledger/fabric/protos/utils"
    42  	"github.com/op/go-logging"
    43  	"golang.org/x/net/context"
    44  	"google.golang.org/grpc"
    45  )
    46  
    47  const ppc64 = "ppc64le"
    48  
    49  const keyfile = "sbft/testdata/key.pem"
    50  const maindir = "github.com/hyperledger/fabric/orderer"
    51  
    52  var ordererDir string
    53  var mainexe string
    54  
    55  type flags struct {
    56  	listenAddr    string
    57  	grpcAddr      string
    58  	telemetryAddr string
    59  	certFile      string
    60  	keyFile       string
    61  	dataDir       string
    62  	genesisFile   string
    63  	verbose       string
    64  	init          string
    65  }
    66  
    67  type Peer struct {
    68  	id     uint64
    69  	config flags
    70  	cancel context.CancelFunc
    71  	cmd    *exec.Cmd
    72  }
    73  
    74  type Receiver struct {
    75  	id      uint64
    76  	retch   chan []byte
    77  	signals chan bool
    78  }
    79  
    80  func skipInShortMode(t *testing.T) {
    81  	if testing.Short() {
    82  		t.Skip("Skipping test in short mode.")
    83  	}
    84  }
    85  
    86  func build() {
    87  	buildcmd := exec.Command("go", "build", "-o", mainexe, maindir)
    88  	buildcmd.Stdout = os.Stdout
    89  	buildcmd.Stderr = os.Stderr
    90  	panicOnError(buildcmd.Run())
    91  }
    92  
    93  func deleteExe() {
    94  	panicOnError(os.Remove(mainexe))
    95  }
    96  
    97  func TestMain(m *testing.M) {
    98  	var err error
    99  	ordererDir, err = os.Getwd()
   100  	if err != nil {
   101  		panic(err)
   102  	}
   103  	mainexe = os.TempDir() + "/" + "orderer"
   104  
   105  	build()
   106  	code := m.Run()
   107  	deleteExe()
   108  	os.Exit(code)
   109  }
   110  
   111  func TestTwoReplicasBroadcastAndDeliverUsingTheSame(t *testing.T) {
   112  	parallelIfNotPpc64(t)
   113  	startingPort := 2000
   114  	skipInShortMode(t)
   115  	peers := InitPeers(2, startingPort)
   116  	StartPeers(peers)
   117  	r, err := Receive(peers[0], startingPort)
   118  	defer r.Stop()
   119  	defer StopPeers(peers)
   120  	if err != nil {
   121  		t.Errorf("Failed to start up receiver: %s", err)
   122  	}
   123  	WaitForConnection(peers)
   124  	if berr := Broadcast(peers[0], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil {
   125  		t.Errorf("Failed to broadcast message: %s", berr)
   126  	}
   127  	if !AssertWithTimeout(func() bool {
   128  		return r.Received() == 2
   129  	}, 30) {
   130  		t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   131  	}
   132  }
   133  
   134  func TestTwoReplicasBroadcastAndDeliverUsingDifferent(t *testing.T) {
   135  	parallelIfNotPpc64(t)
   136  	logging.SetLevel(logging.DEBUG, "sbft")
   137  	startingPort := 2500
   138  	skipInShortMode(t)
   139  	peers := InitPeers(2, startingPort)
   140  	StartPeers(peers)
   141  	r, err := Receive(peers[1], startingPort)
   142  	defer r.Stop()
   143  	defer StopPeers(peers)
   144  	if err != nil {
   145  		t.Errorf("Failed to start up receiver: %s", err)
   146  	}
   147  	WaitForConnection(peers)
   148  	if berr := Broadcast(peers[0], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil {
   149  		t.Errorf("Failed to broadcast message: %s", berr)
   150  	}
   151  	if !AssertWithTimeout(func() bool {
   152  		return r.Received() == 2
   153  	}, 30) {
   154  		t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   155  	}
   156  }
   157  
   158  func TestTenReplicasBroadcastAndDeliverUsingDifferent(t *testing.T) {
   159  	parallelIfNotPpc64(t)
   160  	startingPort := 3000
   161  	skipInShortMode(t)
   162  	peers := InitPeers(10, startingPort)
   163  	StartPeers(peers)
   164  	r, err := Receive(peers[9], startingPort)
   165  	defer r.Stop()
   166  	defer StopPeers(peers)
   167  	if err != nil {
   168  		t.Errorf("Failed to start up receiver: %s", err)
   169  	}
   170  	WaitForConnection(peers)
   171  	if berr := Broadcast(peers[1], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil {
   172  		t.Errorf("Failed to broadcast message: %s", berr)
   173  	}
   174  	if !AssertWithTimeout(func() bool {
   175  		return r.Received() == 2
   176  	}, 30) {
   177  		t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   178  	}
   179  }
   180  
   181  func TestFourReplicasBombedWithBroadcasts(t *testing.T) {
   182  	parallelIfNotPpc64(t)
   183  	startingPort := 4000
   184  	skipInShortMode(t)
   185  	// Add for debug mode:
   186  	// logging.SetLevel(logging.DEBUG, "sbft")
   187  	broadcastCount := 15
   188  	peers := InitPeers(4, startingPort)
   189  	StartPeers(peers)
   190  	r, err := Receive(peers[2], startingPort)
   191  	defer r.Stop()
   192  	defer StopPeers(peers)
   193  	if err != nil {
   194  		t.Errorf("Failed to start up receiver: %s", err)
   195  	}
   196  	WaitForConnection(peers)
   197  	for x := 0; x < broadcastCount; x++ {
   198  		if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil {
   199  			t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x)
   200  		}
   201  		time.Sleep(time.Second)
   202  	}
   203  	if !AssertWithTimeout(func() bool {
   204  		return r.Received() == broadcastCount+1
   205  	}, 30) {
   206  		t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   207  	}
   208  }
   209  
   210  func TestTenReplicasBombedWithBroadcasts(t *testing.T) {
   211  	parallelIfNotPpc64(t)
   212  	startingPort := 5000
   213  	skipInShortMode(t)
   214  	broadcastCount := 15
   215  	peers := InitPeers(10, startingPort)
   216  	StartPeers(peers)
   217  	r, err := Receive(peers[3], startingPort)
   218  	defer r.Stop()
   219  	defer StopPeers(peers)
   220  	if err != nil {
   221  		t.Errorf("Failed to start up receiver: %s", err)
   222  	}
   223  	WaitForConnection(peers)
   224  	for x := 0; x < broadcastCount; x++ {
   225  		if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil {
   226  			t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x)
   227  		}
   228  		time.Sleep(time.Second)
   229  	}
   230  	if !AssertWithTimeout(func() bool {
   231  		return r.Received() == broadcastCount+1
   232  	}, 60) {
   233  		t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   234  	}
   235  }
   236  
   237  func TestTenReplicasBombedWithBroadcastsIfLedgersConsistent(t *testing.T) {
   238  	parallelIfNotPpc64(t)
   239  	startingPort := 6000
   240  	skipInShortMode(t)
   241  	broadcastCount := 15
   242  	peers := InitPeers(10, startingPort)
   243  	StartPeers(peers)
   244  	defer StopPeers(peers)
   245  
   246  	receivers := make([]*Receiver, 0, len(peers))
   247  	for i := 0; i < len(peers); i++ {
   248  		r, err := Receive(peers[i], startingPort)
   249  		if err != nil {
   250  			t.Errorf("Failed to start up receiver: %s", err)
   251  		}
   252  		receivers = append(receivers, r)
   253  	}
   254  
   255  	WaitForConnection(peers)
   256  	for x := 0; x < broadcastCount; x++ {
   257  		if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil {
   258  			t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x)
   259  		}
   260  		time.Sleep(time.Second)
   261  	}
   262  
   263  	for i := 0; i < len(receivers); i++ {
   264  		r := receivers[i]
   265  		if !AssertWithTimeout(func() bool {
   266  			return r.Received() == broadcastCount+1
   267  		}, 60) {
   268  			t.Errorf("Failed to receive some messages. (Received %d)", r.Received())
   269  		}
   270  	}
   271  	for _, r := range receivers {
   272  		r.Stop()
   273  	}
   274  }
   275  
   276  func parallelIfNotPpc64(t *testing.T) {
   277  	if runtime.GOARCH != ppc64 {
   278  		t.Parallel()
   279  	}
   280  }
   281  
   282  func InitPeers(num uint64, startingPort int) []*Peer {
   283  	peers := make([]*Peer, 0, num)
   284  	certFiles := make([]string, 0, num)
   285  	peersWithCerts := map[string]string{}
   286  	for i := uint64(0); i < num; i++ {
   287  		certFile := generateCertificate(i, keyfile)
   288  		certFiles = append(certFiles, certFile)
   289  		peerCommPort := listenAddress(i, startingPort)
   290  		peersWithCerts[peerCommPort] = certFile
   291  	}
   292  	for i := uint64(0); i < num; i++ {
   293  		peerCommPort := listenAddress(i, startingPort)
   294  		grpcPort := grpcPort(i, startingPort)
   295  		configEnv := generateConfigEnv(num, grpcPort, peerCommPort, certFiles[i], peersWithCerts)
   296  		peers = append(peers, initPeer(i, configEnv))
   297  	}
   298  	return peers
   299  }
   300  
   301  func StartPeers(peers []*Peer) {
   302  	for _, p := range peers {
   303  		p.start()
   304  		if runtime.GOARCH == ppc64 {
   305  			time.Sleep(3 * time.Second)
   306  		}
   307  	}
   308  }
   309  
   310  func StopPeers(peers []*Peer) {
   311  	for _, p := range peers {
   312  		p.stop()
   313  	}
   314  }
   315  
   316  func generateConfigEnv(peerNum uint64, grpcPort int, peerCommPort string, certFile string, peersWithCerts map[string]string) []string {
   317  	tempDir, err := ioutil.TempDir("", "sbft_test_config")
   318  	panicOnError(err)
   319  	envs := []string{}
   320  	envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_LOCALMSPDIR=%s", ordererDir+"/../sampleconfig/msp"))
   321  	envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", grpcPort))
   322  	envs = append(envs, fmt.Sprintf("CONFIGTX_ORDERER_ORDERERTYPE=%s", "sbft"))
   323  	envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_GENESISPROFILE=%s", genesisconfig.SampleInsecureProfile))
   324  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_DEPRECATEDBATCHTIMEOUT=%d", 1000))
   325  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_DEPRECATED=%d", 1000000000))
   326  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_REQUESTTIMEOUTNSEC=%d", 1000000000))
   327  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_N=%d", peerNum))
   328  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_F=%d", (peerNum-1)/3))
   329  	js, _ := json.Marshal(peersWithCerts)
   330  	envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_PEERS=%s", js))
   331  	envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_PEERCOMMADDR=%s", peerCommPort))
   332  	envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_CERTFILE=%s", certFile))
   333  	envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_KEYFILE=%s", keyfile))
   334  	envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_DATADIR=%s", tempDir))
   335  	return envs
   336  }
   337  
   338  func initPeer(uid uint64, configEnv []string) (p *Peer) {
   339  	ctx, cancel := context.WithCancel(context.Background())
   340  	p = &Peer{id: uid, cancel: cancel}
   341  	p.cmd = exec.CommandContext(ctx, mainexe)
   342  	p.cmd.Stdout = os.Stdout
   343  	p.cmd.Stderr = os.Stderr
   344  	p.cmd.Env = append(configEnv, os.Environ()...)
   345  	return
   346  }
   347  
   348  func (p *Peer) start() {
   349  	err := p.cmd.Start()
   350  	panicOnError(err)
   351  }
   352  
   353  func (p *Peer) stop() {
   354  	p.cancel()
   355  	p.cmd.Wait()
   356  }
   357  
   358  func Broadcast(p *Peer, startingPort int, bytes []byte) error {
   359  	timeout := 5 * time.Second
   360  	grpcAddress := grpcAddress(p.id, startingPort)
   361  	logger.Warningf("Broadcast - dialing: %s", grpcAddress)
   362  	clientconn, err := grpc.Dial(grpcAddress, grpc.WithBlock(), grpc.WithTimeout(timeout), grpc.WithInsecure())
   363  	if err != nil {
   364  		logger.Warning("Broadcast - failure")
   365  		return err
   366  	}
   367  	defer clientconn.Close()
   368  	client := ab.NewAtomicBroadcastClient(clientconn)
   369  	bstream, err := client.Broadcast(context.Background())
   370  	if err != nil {
   371  		return err
   372  	}
   373  	h := &cb.Header{
   374  		ChannelHeader:   utils.MarshalOrPanic(&cb.ChannelHeader{ChannelId: provisional.TestChainID}),
   375  		SignatureHeader: utils.MarshalOrPanic(&cb.SignatureHeader{})}
   376  	pl := &cb.Payload{Data: bytes, Header: h}
   377  	mpl, err := proto.Marshal(pl)
   378  	panicOnError(err)
   379  	logger.Warningf("Broadcast - sending: %s", grpcAddress)
   380  	if e := bstream.Send(&cb.Envelope{Payload: mpl}); e != nil {
   381  		return e
   382  	}
   383  	_, err = bstream.Recv()
   384  	panicOnError(err)
   385  	logger.Warningf("Broadcast - done: %s", grpcAddress)
   386  	return nil
   387  }
   388  
   389  func Receive(p *Peer, startingPort int) (*Receiver, error) {
   390  	retch := make(chan []byte, 100)
   391  	signals := make(chan bool, 100)
   392  	timeout := 4 * time.Second
   393  	grpcAddress := grpcAddress(p.id, startingPort)
   394  	logger.Warning("Receiver - dial")
   395  	clientconn, err := grpc.Dial(grpcAddress, grpc.WithBlock(), grpc.WithTimeout(timeout), grpc.WithInsecure())
   396  	if err != nil {
   397  		return nil, err
   398  	}
   399  	client := ab.NewAtomicBroadcastClient(clientconn)
   400  	dstream, err := client.Deliver(context.Background())
   401  	if err != nil {
   402  		return nil, err
   403  	}
   404  	dstream.Send(&cb.Envelope{
   405  		Payload: utils.MarshalOrPanic(&cb.Payload{
   406  			Header: &cb.Header{
   407  				ChannelHeader: utils.MarshalOrPanic(&cb.ChannelHeader{
   408  					ChannelId: provisional.TestChainID,
   409  				}),
   410  				SignatureHeader: utils.MarshalOrPanic(&cb.SignatureHeader{}),
   411  			},
   412  
   413  			Data: utils.MarshalOrPanic(&ab.SeekInfo{
   414  				Start:    &ab.SeekPosition{Type: &ab.SeekPosition_Newest{Newest: &ab.SeekNewest{}}},
   415  				Stop:     &ab.SeekPosition{Type: &ab.SeekPosition_Specified{Specified: &ab.SeekSpecified{Number: math.MaxUint64}}},
   416  				Behavior: ab.SeekInfo_BLOCK_UNTIL_READY,
   417  			}),
   418  		}),
   419  	})
   420  
   421  	go func() {
   422  		num := uint64(0)
   423  		for {
   424  			select {
   425  			case <-signals:
   426  				clientconn.Close()
   427  				return
   428  			default:
   429  				m, inerr := dstream.Recv()
   430  				if inerr != nil {
   431  					clientconn.Close()
   432  					return
   433  				}
   434  				b, ok := m.Type.(*ab.DeliverResponse_Block)
   435  				if !ok {
   436  					continue
   437  				}
   438  				for _, tx := range b.Block.Data.Data {
   439  					pl := &cb.Payload{}
   440  					e := &cb.Envelope{}
   441  					merr1 := proto.Unmarshal(tx, e)
   442  					merr2 := proto.Unmarshal(e.Payload, pl)
   443  					if merr1 == nil && merr2 == nil {
   444  						logger.Warning("Receiver - received a message")
   445  						retch <- tx
   446  						num++
   447  					}
   448  				}
   449  			}
   450  		}
   451  	}()
   452  	return &Receiver{id: p.id, retch: retch, signals: signals}, nil
   453  }
   454  
   455  func (r *Receiver) Received() int {
   456  	return len(r.retch)
   457  }
   458  
   459  func (r *Receiver) Stop() {
   460  	close(r.signals)
   461  }
   462  
   463  func AssertWithTimeout(assertion func() bool, timeoutSec int) bool {
   464  	for spent := 0; spent <= timeoutSec && !assertion(); spent++ {
   465  		time.Sleep(time.Second)
   466  	}
   467  	return assertion()
   468  }
   469  
   470  func WaitForConnection(peers []*Peer) {
   471  	l := len(peers)
   472  	m := math.Max(float64(3), float64(l-3))
   473  	_ = <-time.After(time.Duration(m) * time.Second)
   474  }
   475  
   476  func listenAddress(id uint64, startingPort int) string {
   477  	return fmt.Sprintf(":%d", startingPort+2*int(id))
   478  }
   479  
   480  func grpcAddress(id uint64, startingPort int) string {
   481  	return fmt.Sprintf(":%d", grpcPort(id, startingPort))
   482  }
   483  
   484  func grpcPort(id uint64, startingPort int) int {
   485  	return startingPort + 1 + 2*int(id)
   486  }
   487  
   488  func generateCertificate(id uint64, keyFile string) string {
   489  	tempDir, err := ioutil.TempDir("", "sbft_test_cert")
   490  	panicOnError(err)
   491  	readBytes, err := ioutil.ReadFile(keyFile)
   492  	panicOnError(err)
   493  	b, _ := pem.Decode(readBytes)
   494  	priv, err := x509.ParsePKCS1PrivateKey(b.Bytes)
   495  	panicOnError(err)
   496  	notBefore := time.Now()
   497  	notAfter := notBefore.Add(time.Hour)
   498  	template := x509.Certificate{
   499  		SerialNumber: big.NewInt(int64(id)),
   500  		Subject: pkix.Name{
   501  			Organization: []string{"Acme Co"},
   502  		},
   503  		NotBefore: notBefore,
   504  		NotAfter:  notAfter,
   505  
   506  		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
   507  		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
   508  		BasicConstraintsValid: true,
   509  	}
   510  	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
   511  	panicOnError(err)
   512  	certPath := fmt.Sprintf("%s/cert%d.pem", tempDir, id)
   513  	certOut, err := os.Create(certPath)
   514  	panicOnError(err)
   515  	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
   516  	certOut.Close()
   517  	return certPath
   518  }
   519  
   520  func panicOnError(err error) {
   521  	if err != nil {
   522  		panic(err)
   523  	}
   524  }