github.com/leonlxy/hyperledger@v1.0.0-alpha.0.20170427033203-34922035d248/orderer/network_test.go (about) 1 /* 2 Copyright Digital Asset Holdings, LLC 2016 All Rights Reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "crypto/rand" 21 "crypto/x509" 22 "crypto/x509/pkix" 23 "encoding/pem" 24 "fmt" 25 "io/ioutil" 26 "math" 27 "math/big" 28 "os" 29 "os/exec" 30 "runtime" 31 "testing" 32 "time" 33 34 "encoding/json" 35 36 "github.com/golang/protobuf/proto" 37 genesisconfig "github.com/hyperledger/fabric/common/configtx/tool/localconfig" 38 "github.com/hyperledger/fabric/common/configtx/tool/provisional" 39 cb "github.com/hyperledger/fabric/protos/common" 40 ab "github.com/hyperledger/fabric/protos/orderer" 41 "github.com/hyperledger/fabric/protos/utils" 42 "github.com/op/go-logging" 43 "golang.org/x/net/context" 44 "google.golang.org/grpc" 45 ) 46 47 const ppc64 = "ppc64le" 48 49 const keyfile = "sbft/testdata/key.pem" 50 const maindir = "github.com/hyperledger/fabric/orderer" 51 52 var ordererDir string 53 var mainexe string 54 55 type flags struct { 56 listenAddr string 57 grpcAddr string 58 telemetryAddr string 59 certFile string 60 keyFile string 61 dataDir string 62 genesisFile string 63 verbose string 64 init string 65 } 66 67 type Peer struct { 68 id uint64 69 config flags 70 cancel context.CancelFunc 71 cmd *exec.Cmd 72 } 73 74 type Receiver struct { 75 id uint64 76 retch chan []byte 77 signals chan bool 78 } 79 80 func skipInShortMode(t *testing.T) { 81 if testing.Short() { 82 t.Skip("Skipping test in short mode.") 83 } 84 } 85 86 func build() { 87 buildcmd := exec.Command("go", "build", "-o", mainexe, maindir) 88 buildcmd.Stdout = os.Stdout 89 buildcmd.Stderr = os.Stderr 90 panicOnError(buildcmd.Run()) 91 } 92 93 func deleteExe() { 94 panicOnError(os.Remove(mainexe)) 95 } 96 97 func TestMain(m *testing.M) { 98 var err error 99 ordererDir, err = os.Getwd() 100 if err != nil { 101 panic(err) 102 } 103 mainexe = os.TempDir() + "/" + "orderer" 104 105 build() 106 code := m.Run() 107 deleteExe() 108 os.Exit(code) 109 } 110 111 func TestTwoReplicasBroadcastAndDeliverUsingTheSame(t *testing.T) { 112 parallelIfNotPpc64(t) 113 startingPort := 2000 114 skipInShortMode(t) 115 peers := InitPeers(2, startingPort) 116 StartPeers(peers) 117 r, err := Receive(peers[0], startingPort) 118 defer r.Stop() 119 defer StopPeers(peers) 120 if err != nil { 121 t.Errorf("Failed to start up receiver: %s", err) 122 } 123 WaitForConnection(peers) 124 if berr := Broadcast(peers[0], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil { 125 t.Errorf("Failed to broadcast message: %s", berr) 126 } 127 if !AssertWithTimeout(func() bool { 128 return r.Received() == 2 129 }, 30) { 130 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 131 } 132 } 133 134 func TestTwoReplicasBroadcastAndDeliverUsingDifferent(t *testing.T) { 135 parallelIfNotPpc64(t) 136 logging.SetLevel(logging.DEBUG, "sbft") 137 startingPort := 2500 138 skipInShortMode(t) 139 peers := InitPeers(2, startingPort) 140 StartPeers(peers) 141 r, err := Receive(peers[1], startingPort) 142 defer r.Stop() 143 defer StopPeers(peers) 144 if err != nil { 145 t.Errorf("Failed to start up receiver: %s", err) 146 } 147 WaitForConnection(peers) 148 if berr := Broadcast(peers[0], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil { 149 t.Errorf("Failed to broadcast message: %s", berr) 150 } 151 if !AssertWithTimeout(func() bool { 152 return r.Received() == 2 153 }, 30) { 154 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 155 } 156 } 157 158 func TestTenReplicasBroadcastAndDeliverUsingDifferent(t *testing.T) { 159 parallelIfNotPpc64(t) 160 startingPort := 3000 161 skipInShortMode(t) 162 peers := InitPeers(10, startingPort) 163 StartPeers(peers) 164 r, err := Receive(peers[9], startingPort) 165 defer r.Stop() 166 defer StopPeers(peers) 167 if err != nil { 168 t.Errorf("Failed to start up receiver: %s", err) 169 } 170 WaitForConnection(peers) 171 if berr := Broadcast(peers[1], startingPort, []byte{0, 1, 2, 3, 4}); berr != nil { 172 t.Errorf("Failed to broadcast message: %s", berr) 173 } 174 if !AssertWithTimeout(func() bool { 175 return r.Received() == 2 176 }, 30) { 177 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 178 } 179 } 180 181 func TestFourReplicasBombedWithBroadcasts(t *testing.T) { 182 parallelIfNotPpc64(t) 183 startingPort := 4000 184 skipInShortMode(t) 185 // Add for debug mode: 186 // logging.SetLevel(logging.DEBUG, "sbft") 187 broadcastCount := 15 188 peers := InitPeers(4, startingPort) 189 StartPeers(peers) 190 r, err := Receive(peers[2], startingPort) 191 defer r.Stop() 192 defer StopPeers(peers) 193 if err != nil { 194 t.Errorf("Failed to start up receiver: %s", err) 195 } 196 WaitForConnection(peers) 197 for x := 0; x < broadcastCount; x++ { 198 if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil { 199 t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x) 200 } 201 time.Sleep(time.Second) 202 } 203 if !AssertWithTimeout(func() bool { 204 return r.Received() == broadcastCount+1 205 }, 30) { 206 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 207 } 208 } 209 210 func TestTenReplicasBombedWithBroadcasts(t *testing.T) { 211 parallelIfNotPpc64(t) 212 startingPort := 5000 213 skipInShortMode(t) 214 broadcastCount := 15 215 peers := InitPeers(10, startingPort) 216 StartPeers(peers) 217 r, err := Receive(peers[3], startingPort) 218 defer r.Stop() 219 defer StopPeers(peers) 220 if err != nil { 221 t.Errorf("Failed to start up receiver: %s", err) 222 } 223 WaitForConnection(peers) 224 for x := 0; x < broadcastCount; x++ { 225 if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil { 226 t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x) 227 } 228 time.Sleep(time.Second) 229 } 230 if !AssertWithTimeout(func() bool { 231 return r.Received() == broadcastCount+1 232 }, 60) { 233 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 234 } 235 } 236 237 func TestTenReplicasBombedWithBroadcastsIfLedgersConsistent(t *testing.T) { 238 parallelIfNotPpc64(t) 239 startingPort := 6000 240 skipInShortMode(t) 241 broadcastCount := 15 242 peers := InitPeers(10, startingPort) 243 StartPeers(peers) 244 defer StopPeers(peers) 245 246 receivers := make([]*Receiver, 0, len(peers)) 247 for i := 0; i < len(peers); i++ { 248 r, err := Receive(peers[i], startingPort) 249 if err != nil { 250 t.Errorf("Failed to start up receiver: %s", err) 251 } 252 receivers = append(receivers, r) 253 } 254 255 WaitForConnection(peers) 256 for x := 0; x < broadcastCount; x++ { 257 if berr := Broadcast(peers[2], startingPort, []byte{0, 1, 2, byte(x), 3, 4, byte(x)}); berr != nil { 258 t.Errorf("Failed to broadcast message: %s (broadcast number %d)", berr, x) 259 } 260 time.Sleep(time.Second) 261 } 262 263 for i := 0; i < len(receivers); i++ { 264 r := receivers[i] 265 if !AssertWithTimeout(func() bool { 266 return r.Received() == broadcastCount+1 267 }, 60) { 268 t.Errorf("Failed to receive some messages. (Received %d)", r.Received()) 269 } 270 } 271 for _, r := range receivers { 272 r.Stop() 273 } 274 } 275 276 func parallelIfNotPpc64(t *testing.T) { 277 if runtime.GOARCH != ppc64 { 278 t.Parallel() 279 } 280 } 281 282 func InitPeers(num uint64, startingPort int) []*Peer { 283 peers := make([]*Peer, 0, num) 284 certFiles := make([]string, 0, num) 285 peersWithCerts := map[string]string{} 286 for i := uint64(0); i < num; i++ { 287 certFile := generateCertificate(i, keyfile) 288 certFiles = append(certFiles, certFile) 289 peerCommPort := listenAddress(i, startingPort) 290 peersWithCerts[peerCommPort] = certFile 291 } 292 for i := uint64(0); i < num; i++ { 293 peerCommPort := listenAddress(i, startingPort) 294 grpcPort := grpcPort(i, startingPort) 295 configEnv := generateConfigEnv(num, grpcPort, peerCommPort, certFiles[i], peersWithCerts) 296 peers = append(peers, initPeer(i, configEnv)) 297 } 298 return peers 299 } 300 301 func StartPeers(peers []*Peer) { 302 for _, p := range peers { 303 p.start() 304 if runtime.GOARCH == ppc64 { 305 time.Sleep(3 * time.Second) 306 } 307 } 308 } 309 310 func StopPeers(peers []*Peer) { 311 for _, p := range peers { 312 p.stop() 313 } 314 } 315 316 func generateConfigEnv(peerNum uint64, grpcPort int, peerCommPort string, certFile string, peersWithCerts map[string]string) []string { 317 tempDir, err := ioutil.TempDir("", "sbft_test_config") 318 panicOnError(err) 319 envs := []string{} 320 envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_LOCALMSPDIR=%s", ordererDir+"/../sampleconfig/msp")) 321 envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", grpcPort)) 322 envs = append(envs, fmt.Sprintf("CONFIGTX_ORDERER_ORDERERTYPE=%s", "sbft")) 323 envs = append(envs, fmt.Sprintf("ORDERER_GENERAL_GENESISPROFILE=%s", genesisconfig.SampleInsecureProfile)) 324 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_DEPRECATEDBATCHTIMEOUT=%d", 1000)) 325 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_DEPRECATED=%d", 1000000000)) 326 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_REQUESTTIMEOUTNSEC=%d", 1000000000)) 327 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_N=%d", peerNum)) 328 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_F=%d", (peerNum-1)/3)) 329 js, _ := json.Marshal(peersWithCerts) 330 envs = append(envs, fmt.Sprintf("ORDERER_GENESIS_SBFTSHARED_PEERS=%s", js)) 331 envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_PEERCOMMADDR=%s", peerCommPort)) 332 envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_CERTFILE=%s", certFile)) 333 envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_KEYFILE=%s", keyfile)) 334 envs = append(envs, fmt.Sprintf("ORDERER_SBFTLOCAL_DATADIR=%s", tempDir)) 335 return envs 336 } 337 338 func initPeer(uid uint64, configEnv []string) (p *Peer) { 339 ctx, cancel := context.WithCancel(context.Background()) 340 p = &Peer{id: uid, cancel: cancel} 341 p.cmd = exec.CommandContext(ctx, mainexe) 342 p.cmd.Stdout = os.Stdout 343 p.cmd.Stderr = os.Stderr 344 p.cmd.Env = append(configEnv, os.Environ()...) 345 return 346 } 347 348 func (p *Peer) start() { 349 err := p.cmd.Start() 350 panicOnError(err) 351 } 352 353 func (p *Peer) stop() { 354 p.cancel() 355 p.cmd.Wait() 356 } 357 358 func Broadcast(p *Peer, startingPort int, bytes []byte) error { 359 timeout := 5 * time.Second 360 grpcAddress := grpcAddress(p.id, startingPort) 361 logger.Warningf("Broadcast - dialing: %s", grpcAddress) 362 clientconn, err := grpc.Dial(grpcAddress, grpc.WithBlock(), grpc.WithTimeout(timeout), grpc.WithInsecure()) 363 if err != nil { 364 logger.Warning("Broadcast - failure") 365 return err 366 } 367 defer clientconn.Close() 368 client := ab.NewAtomicBroadcastClient(clientconn) 369 bstream, err := client.Broadcast(context.Background()) 370 if err != nil { 371 return err 372 } 373 h := &cb.Header{ 374 ChannelHeader: utils.MarshalOrPanic(&cb.ChannelHeader{ChannelId: provisional.TestChainID}), 375 SignatureHeader: utils.MarshalOrPanic(&cb.SignatureHeader{})} 376 pl := &cb.Payload{Data: bytes, Header: h} 377 mpl, err := proto.Marshal(pl) 378 panicOnError(err) 379 logger.Warningf("Broadcast - sending: %s", grpcAddress) 380 if e := bstream.Send(&cb.Envelope{Payload: mpl}); e != nil { 381 return e 382 } 383 _, err = bstream.Recv() 384 panicOnError(err) 385 logger.Warningf("Broadcast - done: %s", grpcAddress) 386 return nil 387 } 388 389 func Receive(p *Peer, startingPort int) (*Receiver, error) { 390 retch := make(chan []byte, 100) 391 signals := make(chan bool, 100) 392 timeout := 4 * time.Second 393 grpcAddress := grpcAddress(p.id, startingPort) 394 logger.Warning("Receiver - dial") 395 clientconn, err := grpc.Dial(grpcAddress, grpc.WithBlock(), grpc.WithTimeout(timeout), grpc.WithInsecure()) 396 if err != nil { 397 return nil, err 398 } 399 client := ab.NewAtomicBroadcastClient(clientconn) 400 dstream, err := client.Deliver(context.Background()) 401 if err != nil { 402 return nil, err 403 } 404 dstream.Send(&cb.Envelope{ 405 Payload: utils.MarshalOrPanic(&cb.Payload{ 406 Header: &cb.Header{ 407 ChannelHeader: utils.MarshalOrPanic(&cb.ChannelHeader{ 408 ChannelId: provisional.TestChainID, 409 }), 410 SignatureHeader: utils.MarshalOrPanic(&cb.SignatureHeader{}), 411 }, 412 413 Data: utils.MarshalOrPanic(&ab.SeekInfo{ 414 Start: &ab.SeekPosition{Type: &ab.SeekPosition_Newest{Newest: &ab.SeekNewest{}}}, 415 Stop: &ab.SeekPosition{Type: &ab.SeekPosition_Specified{Specified: &ab.SeekSpecified{Number: math.MaxUint64}}}, 416 Behavior: ab.SeekInfo_BLOCK_UNTIL_READY, 417 }), 418 }), 419 }) 420 421 go func() { 422 num := uint64(0) 423 for { 424 select { 425 case <-signals: 426 clientconn.Close() 427 return 428 default: 429 m, inerr := dstream.Recv() 430 if inerr != nil { 431 clientconn.Close() 432 return 433 } 434 b, ok := m.Type.(*ab.DeliverResponse_Block) 435 if !ok { 436 continue 437 } 438 for _, tx := range b.Block.Data.Data { 439 pl := &cb.Payload{} 440 e := &cb.Envelope{} 441 merr1 := proto.Unmarshal(tx, e) 442 merr2 := proto.Unmarshal(e.Payload, pl) 443 if merr1 == nil && merr2 == nil { 444 logger.Warning("Receiver - received a message") 445 retch <- tx 446 num++ 447 } 448 } 449 } 450 } 451 }() 452 return &Receiver{id: p.id, retch: retch, signals: signals}, nil 453 } 454 455 func (r *Receiver) Received() int { 456 return len(r.retch) 457 } 458 459 func (r *Receiver) Stop() { 460 close(r.signals) 461 } 462 463 func AssertWithTimeout(assertion func() bool, timeoutSec int) bool { 464 for spent := 0; spent <= timeoutSec && !assertion(); spent++ { 465 time.Sleep(time.Second) 466 } 467 return assertion() 468 } 469 470 func WaitForConnection(peers []*Peer) { 471 l := len(peers) 472 m := math.Max(float64(3), float64(l-3)) 473 _ = <-time.After(time.Duration(m) * time.Second) 474 } 475 476 func listenAddress(id uint64, startingPort int) string { 477 return fmt.Sprintf(":%d", startingPort+2*int(id)) 478 } 479 480 func grpcAddress(id uint64, startingPort int) string { 481 return fmt.Sprintf(":%d", grpcPort(id, startingPort)) 482 } 483 484 func grpcPort(id uint64, startingPort int) int { 485 return startingPort + 1 + 2*int(id) 486 } 487 488 func generateCertificate(id uint64, keyFile string) string { 489 tempDir, err := ioutil.TempDir("", "sbft_test_cert") 490 panicOnError(err) 491 readBytes, err := ioutil.ReadFile(keyFile) 492 panicOnError(err) 493 b, _ := pem.Decode(readBytes) 494 priv, err := x509.ParsePKCS1PrivateKey(b.Bytes) 495 panicOnError(err) 496 notBefore := time.Now() 497 notAfter := notBefore.Add(time.Hour) 498 template := x509.Certificate{ 499 SerialNumber: big.NewInt(int64(id)), 500 Subject: pkix.Name{ 501 Organization: []string{"Acme Co"}, 502 }, 503 NotBefore: notBefore, 504 NotAfter: notAfter, 505 506 KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 507 ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 508 BasicConstraintsValid: true, 509 } 510 derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) 511 panicOnError(err) 512 certPath := fmt.Sprintf("%s/cert%d.pem", tempDir, id) 513 certOut, err := os.Create(certPath) 514 panicOnError(err) 515 pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) 516 certOut.Close() 517 return certPath 518 } 519 520 func panicOnError(err error) { 521 if err != nil { 522 panic(err) 523 } 524 }