github.com/leonlxy/hyperledger@v1.0.0-alpha.0.20170427033203-34922035d248/gossip/state/state.go (about) 1 /* 2 Copyright IBM Corp. 2016 All Rights Reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package state 18 19 import ( 20 "bytes" 21 "errors" 22 "sync" 23 "sync/atomic" 24 "time" 25 26 pb "github.com/golang/protobuf/proto" 27 "github.com/hyperledger/fabric/core/committer" 28 "github.com/hyperledger/fabric/gossip/api" 29 "github.com/hyperledger/fabric/gossip/comm" 30 common2 "github.com/hyperledger/fabric/gossip/common" 31 "github.com/hyperledger/fabric/gossip/discovery" 32 "github.com/hyperledger/fabric/gossip/util" 33 "github.com/hyperledger/fabric/protos/common" 34 proto "github.com/hyperledger/fabric/protos/gossip" 35 "github.com/op/go-logging" 36 ) 37 38 // GossipStateProvider is the interface to acquire sequences of the ledger blocks 39 // capable to full fill missing blocks by running state replication and 40 // sending request to get missing block to other nodes 41 type GossipStateProvider interface { 42 // Retrieve block with sequence number equal to index 43 GetBlock(index uint64) *common.Block 44 45 AddPayload(payload *proto.Payload) error 46 47 // Stop terminates state transfer object 48 Stop() 49 } 50 51 const ( 52 defAntiEntropyInterval = 10 * time.Second 53 defAntiEntropyStateResponseTimeout = 3 * time.Second 54 defAntiEntropyBatchSize = 10 55 56 defChannelBufferSize = 100 57 defAntiEntropyMaxRetries = 3 58 ) 59 60 // GossipAdapter defines gossip/communication required interface for state provider 61 type GossipAdapter interface { 62 // Send sends a message to remote peers 63 Send(msg *proto.GossipMessage, peers ...*comm.RemotePeer) 64 65 // Accept returns a dedicated read-only channel for messages sent by other nodes that match a certain predicate. 66 // If passThrough is false, the messages are processed by the gossip layer beforehand. 67 // If passThrough is true, the gossip layer doesn't intervene and the messages 68 // can be used to send a reply back to the sender 69 Accept(acceptor common2.MessageAcceptor, passThrough bool) (<-chan *proto.GossipMessage, <-chan proto.ReceivedMessage) 70 71 // UpdateChannelMetadata updates the self metadata the peer 72 // publishes to other peers about its channel-related state 73 UpdateChannelMetadata(metadata []byte, chainID common2.ChainID) 74 75 // PeersOfChannel returns the NetworkMembers considered alive 76 // and also subscribed to the channel given 77 PeersOfChannel(common2.ChainID) []discovery.NetworkMember 78 } 79 80 // GossipStateProviderImpl the implementation of the GossipStateProvider interface 81 // the struct to handle in memory sliding window of 82 // new ledger block to be acquired by hyper ledger 83 type GossipStateProviderImpl struct { 84 // MessageCryptoService 85 mcs api.MessageCryptoService 86 87 // Chain id 88 chainID string 89 90 // The gossiping service 91 gossip GossipAdapter 92 93 // Channel to read gossip messages from 94 gossipChan <-chan *proto.GossipMessage 95 96 commChan <-chan proto.ReceivedMessage 97 98 mutex sync.RWMutex 99 100 // Queue of payloads which wasn't acquired yet 101 payloads PayloadsBuffer 102 103 committer committer.Committer 104 105 stateResponseCh chan proto.ReceivedMessage 106 107 stateRequestCh chan proto.ReceivedMessage 108 109 stateTransferActive int32 110 111 stopCh chan struct{} 112 113 done sync.WaitGroup 114 115 once sync.Once 116 } 117 118 var logger *logging.Logger // package-level logger 119 120 func init() { 121 logger = util.GetLogger(util.LoggingStateModule, "") 122 } 123 124 // NewGossipStateProvider creates initialized instance of gossip state provider 125 func NewGossipStateProvider(chainID string, g GossipAdapter, committer committer.Committer, mcs api.MessageCryptoService) GossipStateProvider { 126 logger := util.GetLogger(util.LoggingStateModule, "") 127 128 gossipChan, _ := g.Accept(func(message interface{}) bool { 129 // Get only data messages 130 return message.(*proto.GossipMessage).IsDataMsg() && 131 bytes.Equal(message.(*proto.GossipMessage).Channel, []byte(chainID)) 132 }, false) 133 134 remoteStateMsgFilter := func(message interface{}) bool { 135 receivedMsg := message.(proto.ReceivedMessage) 136 msg := receivedMsg.GetGossipMessage() 137 if !msg.IsRemoteStateMessage() { 138 return false 139 } 140 // If we're not running with authentication, no point 141 // in enforcing access control 142 if !receivedMsg.GetConnectionInfo().IsAuthenticated() { 143 return true 144 } 145 connInfo := receivedMsg.GetConnectionInfo() 146 authErr := mcs.VerifyByChannel(msg.Channel, connInfo.Identity, connInfo.Auth.Signature, connInfo.Auth.SignedData) 147 if authErr != nil { 148 logger.Warning("Got unauthorized nodeMetastate transfer request from", string(connInfo.Identity)) 149 return false 150 } 151 return true 152 } 153 154 // Filter message which are only relevant for nodeMetastate transfer 155 _, commChan := g.Accept(remoteStateMsgFilter, true) 156 157 height, err := committer.LedgerHeight() 158 if height == 0 { 159 // Panic here since this is an indication of invalid situation which should not happen in normal 160 // code path. 161 logger.Panic("Committer height cannot be zero, ledger should include at least one block (genesis).") 162 } 163 164 if err != nil { 165 logger.Error("Could not read ledger info to obtain current ledger height due to: ", err) 166 // Exiting as without ledger it will be impossible 167 // to deliver new blocks 168 return nil 169 } 170 171 s := &GossipStateProviderImpl{ 172 // MessageCryptoService 173 mcs: mcs, 174 175 // Chain ID 176 chainID: chainID, 177 178 // Instance of the gossip 179 gossip: g, 180 181 // Channel to read new messages from 182 gossipChan: gossipChan, 183 184 // Channel to read direct messages from other peers 185 commChan: commChan, 186 187 // Create a queue for payload received 188 payloads: NewPayloadsBuffer(height), 189 190 committer: committer, 191 192 stateResponseCh: make(chan proto.ReceivedMessage, defChannelBufferSize), 193 194 stateRequestCh: make(chan proto.ReceivedMessage, defChannelBufferSize), 195 196 stopCh: make(chan struct{}, 1), 197 198 stateTransferActive: 0, 199 200 once: sync.Once{}, 201 } 202 203 nodeMetastate := NewNodeMetastate(height - 1) 204 205 logger.Infof("Updating node metadata information, "+ 206 "current ledger sequence is at = %d, next expected block is = %d", nodeMetastate.LedgerHeight, s.payloads.Next()) 207 208 b, err := nodeMetastate.Bytes() 209 if err == nil { 210 logger.Debug("Updating gossip metadate nodeMetastate", nodeMetastate) 211 g.UpdateChannelMetadata(b, common2.ChainID(s.chainID)) 212 } else { 213 logger.Errorf("Unable to serialize node meta nodeMetastate, error = %s", err) 214 } 215 216 s.done.Add(4) 217 218 // Listen for incoming communication 219 go s.listen() 220 // Deliver in order messages into the incoming channel 221 go s.deliverPayloads() 222 // Execute anti entropy to fill missing gaps 223 go s.antiEntropy() 224 // Taking care of state request messages 225 go s.processStateRequests() 226 227 return s 228 } 229 230 func (s *GossipStateProviderImpl) listen() { 231 defer s.done.Done() 232 233 for { 234 select { 235 case msg := <-s.gossipChan: 236 logger.Debug("Received new message via gossip channel") 237 go s.queueNewMessage(msg) 238 case msg := <-s.commChan: 239 logger.Debug("Direct message ", msg) 240 go s.directMessage(msg) 241 case <-s.stopCh: 242 s.stopCh <- struct{}{} 243 logger.Debug("Stop listening for new messages") 244 return 245 } 246 } 247 } 248 249 func (s *GossipStateProviderImpl) directMessage(msg proto.ReceivedMessage) { 250 logger.Debug("[ENTER] -> directMessage") 251 defer logger.Debug("[EXIT] -> directMessage") 252 253 if msg == nil { 254 logger.Error("Got nil message via end-to-end channel, should not happen!") 255 return 256 } 257 258 if !bytes.Equal(msg.GetGossipMessage().Channel, []byte(s.chainID)) { 259 logger.Warning("Received state transfer request for channel", 260 string(msg.GetGossipMessage().Channel), "while expecting channel", s.chainID, "skipping request...") 261 return 262 } 263 264 incoming := msg.GetGossipMessage() 265 266 if incoming.GetStateRequest() != nil { 267 if len(s.stateRequestCh) < defChannelBufferSize { 268 // Forward state request to the channel, if there are too 269 // many message of state request ignore to avoid flooding. 270 s.stateRequestCh <- msg 271 } 272 } else if incoming.GetStateResponse() != nil { 273 // If no state transfer procedure activate there is 274 // no reason to process the message 275 if atomic.LoadInt32(&s.stateTransferActive) == 1 { 276 // Send signal of state response message 277 s.stateResponseCh <- msg 278 } 279 } 280 } 281 282 func (s *GossipStateProviderImpl) processStateRequests() { 283 defer s.done.Done() 284 285 for { 286 select { 287 case msg := <-s.stateRequestCh: 288 s.handleStateRequest(msg) 289 case <-s.stopCh: 290 s.stopCh <- struct{}{} 291 return 292 } 293 } 294 } 295 296 // Handle state request message, validate batch size, read current leader state to 297 // obtain required blocks, build response message and send it back 298 func (s *GossipStateProviderImpl) handleStateRequest(msg proto.ReceivedMessage) { 299 if msg == nil { 300 return 301 } 302 request := msg.GetGossipMessage().GetStateRequest() 303 304 batchSize := request.EndSeqNum - request.StartSeqNum 305 if batchSize > defAntiEntropyBatchSize { 306 logger.Errorf("Requesting blocks batchSize size (%d) greater than configured allowed"+ 307 " (%d) batching for anti-entropy. Ignoring request...", batchSize, defAntiEntropyBatchSize) 308 return 309 } 310 311 if request.StartSeqNum > request.EndSeqNum { 312 logger.Errorf("Invalid sequence interval [%d...%d], ignoring request...", request.StartSeqNum, request.EndSeqNum) 313 return 314 } 315 316 currentHeight, err := s.committer.LedgerHeight() 317 if err != nil { 318 logger.Errorf("Cannot access to current ledger height, due to %s", err) 319 return 320 } 321 if currentHeight < request.EndSeqNum { 322 logger.Warningf("Received state request to transfer blocks with sequence numbers higher [%d...%d] "+ 323 "than available in ledger (%d)", request.StartSeqNum, request.StartSeqNum, currentHeight) 324 } 325 326 endSeqNum := min(currentHeight, request.EndSeqNum) 327 328 response := &proto.RemoteStateResponse{Payloads: make([]*proto.Payload, 0)} 329 for seqNum := request.StartSeqNum; seqNum <= endSeqNum; seqNum++ { 330 logger.Debug("Reading block ", seqNum, " from the committer service") 331 blocks := s.committer.GetBlocks([]uint64{seqNum}) 332 333 if len(blocks) == 0 { 334 logger.Errorf("Wasn't able to read block with sequence number %d from ledger, skipping....", seqNum) 335 continue 336 } 337 338 blockBytes, err := pb.Marshal(blocks[0]) 339 if err != nil { 340 logger.Errorf("Could not marshal block: %s", err) 341 } 342 343 response.Payloads = append(response.Payloads, &proto.Payload{ 344 SeqNum: seqNum, 345 Data: blockBytes, 346 Hash: string(blocks[0].Header.Hash()), 347 }) 348 } 349 // Sending back response with missing blocks 350 msg.Respond(&proto.GossipMessage{ 351 // Copy nonce field from the request, so it will be possible to match response 352 Nonce: msg.GetGossipMessage().Nonce, 353 Tag: proto.GossipMessage_CHAN_OR_ORG, 354 Channel: []byte(s.chainID), 355 Content: &proto.GossipMessage_StateResponse{response}, 356 }) 357 } 358 359 func (s *GossipStateProviderImpl) handleStateResponse(msg proto.ReceivedMessage) (uint64, error) { 360 max := uint64(0) 361 // Send signal that response for given nonce has been received 362 response := msg.GetGossipMessage().GetStateResponse() 363 // Extract payloads, verify and push into buffer 364 if len(response.GetPayloads()) == 0 { 365 return uint64(0), errors.New("Received state tranfer response without payload") 366 } 367 for _, payload := range response.GetPayloads() { 368 logger.Debugf("Received payload with sequence number %d.", payload.SeqNum) 369 if err := s.mcs.VerifyBlock(common2.ChainID(s.chainID), payload.Data); err != nil { 370 logger.Warningf("Error verifying block with sequence number %d, due to %s", payload.SeqNum, err) 371 return uint64(0), err 372 } 373 if max < payload.SeqNum { 374 max = payload.SeqNum 375 } 376 err := s.payloads.Push(payload) 377 if err != nil { 378 logger.Warningf("Payload with sequence number %d was received earlier", payload.SeqNum) 379 } 380 } 381 return max, nil 382 } 383 384 // Stop function send halting signal to all go routines 385 func (s *GossipStateProviderImpl) Stop() { 386 // Make sure stop won't be executed twice 387 // and stop channel won't be used again 388 s.once.Do(func() { 389 s.stopCh <- struct{}{} 390 // Make sure all go-routines has finished 391 s.done.Wait() 392 // Close all resources 393 s.committer.Close() 394 close(s.stateRequestCh) 395 close(s.stateResponseCh) 396 close(s.stopCh) 397 }) 398 } 399 400 // New message notification/handler 401 func (s *GossipStateProviderImpl) queueNewMessage(msg *proto.GossipMessage) { 402 if !bytes.Equal(msg.Channel, []byte(s.chainID)) { 403 logger.Warning("Received state transfer request for channel", 404 string(msg.Channel), "while expecting channel", s.chainID, "skipping request...") 405 return 406 } 407 408 dataMsg := msg.GetDataMsg() 409 if dataMsg != nil { 410 // Add new payload to ordered set 411 412 logger.Debugf("Received new payload with sequence number = [%d]", dataMsg.Payload.SeqNum) 413 s.payloads.Push(dataMsg.GetPayload()) 414 } else { 415 logger.Debug("Gossip message received is not of data message type, usually this should not happen.") 416 } 417 } 418 419 func (s *GossipStateProviderImpl) deliverPayloads() { 420 defer s.done.Done() 421 422 for { 423 select { 424 // Wait for notification that next seq has arrived 425 case <-s.payloads.Ready(): 426 logger.Debugf("Ready to transfer payloads to the ledger, next sequence number is = [%d]", s.payloads.Next()) 427 // Collect all subsequent payloads 428 for payload := s.payloads.Pop(); payload != nil; payload = s.payloads.Pop() { 429 rawblock := &common.Block{} 430 if err := pb.Unmarshal(payload.Data, rawblock); err != nil { 431 logger.Errorf("Error getting block with seqNum = %d due to (%s)...dropping block", payload.SeqNum, err) 432 continue 433 } 434 logger.Debug("New block with sequence number ", payload.SeqNum, " transactions num ", len(rawblock.Data.Data)) 435 s.commitBlock(rawblock) 436 } 437 case <-s.stopCh: 438 s.stopCh <- struct{}{} 439 logger.Debug("State provider has been stoped, finishing to push new blocks.") 440 return 441 } 442 } 443 } 444 445 func (s *GossipStateProviderImpl) antiEntropy() { 446 defer s.done.Done() 447 defer logger.Debug("State Provider stopped, stopping anti entropy procedure.") 448 449 for { 450 select { 451 case <-s.stopCh: 452 s.stopCh <- struct{}{} 453 return 454 case <-time.After(defAntiEntropyInterval): 455 current, err := s.committer.LedgerHeight() 456 if err != nil { 457 // Unable to read from ledger continue to the next round 458 logger.Error("Cannot obtain ledger height, due to", err) 459 continue 460 } 461 max := s.maxAvailableLedgerHeight() 462 463 if current == max { 464 continue 465 } 466 467 s.requestBlocksInRange(uint64(current), uint64(max)) 468 } 469 } 470 } 471 472 // Iterate over all available peers and check advertised meta state to 473 // find maximum available ledger height across peers 474 func (s *GossipStateProviderImpl) maxAvailableLedgerHeight() uint64 { 475 max := uint64(0) 476 for _, p := range s.gossip.PeersOfChannel(common2.ChainID(s.chainID)) { 477 if nodeMetastate, err := FromBytes(p.Metadata); err == nil { 478 if max < nodeMetastate.LedgerHeight { 479 max = nodeMetastate.LedgerHeight 480 } 481 } 482 } 483 return max 484 } 485 486 // GetBlocksInRange capable to acquire blocks with sequence 487 // numbers in the range [start...end]. 488 func (s *GossipStateProviderImpl) requestBlocksInRange(start uint64, end uint64) { 489 atomic.StoreInt32(&s.stateTransferActive, 1) 490 defer atomic.StoreInt32(&s.stateTransferActive, 0) 491 492 for prev := start; prev <= end; { 493 next := min(end, prev+defAntiEntropyBatchSize) 494 495 gossipMsg := s.stateRequestMessage(prev, next) 496 497 responseReceived := false 498 tryCounts := 0 499 500 for !responseReceived { 501 if tryCounts > defAntiEntropyMaxRetries { 502 logger.Warningf("Wasn't able to get blocks in range [%d...%d], after %d retries", 503 prev, next, tryCounts) 504 return 505 } 506 // Select peers to ask for blocks 507 peer, err := s.selectPeerToRequestFrom(next) 508 if err != nil { 509 logger.Warningf("Cannot send state request for blocks in range [%d...%d], due to", 510 prev, next, err) 511 return 512 } 513 514 logger.Debugf("State transfer, with peer %s, requesting blocks in range [%d...%d], "+ 515 "for chainID %s", peer.Endpoint, prev, next, s.chainID) 516 517 s.gossip.Send(gossipMsg, peer) 518 tryCounts++ 519 520 // Wait until timeout or response arrival 521 select { 522 case msg := <-s.stateResponseCh: 523 if msg.GetGossipMessage().Nonce != gossipMsg.Nonce { 524 continue 525 } 526 // Got corresponding response for state request, can continue 527 index, err := s.handleStateResponse(msg) 528 if err != nil { 529 logger.Warningf("Wasn't able to process state response for "+ 530 "blocks [%d...%d], due to %s", prev, next, err) 531 continue 532 } 533 prev = index + 1 534 responseReceived = true 535 case <-time.After(defAntiEntropyStateResponseTimeout): 536 case <-s.stopCh: 537 s.stopCh <- struct{}{} 538 return 539 } 540 } 541 } 542 } 543 544 // Generate state request message for given blocks in range [beginSeq...endSeq] 545 func (s *GossipStateProviderImpl) stateRequestMessage(beginSeq uint64, endSeq uint64) *proto.GossipMessage { 546 return &proto.GossipMessage{ 547 Nonce: util.RandomUInt64(), 548 Tag: proto.GossipMessage_CHAN_OR_ORG, 549 Channel: []byte(s.chainID), 550 Content: &proto.GossipMessage_StateRequest{ 551 StateRequest: &proto.RemoteStateRequest{ 552 StartSeqNum: beginSeq, 553 EndSeqNum: endSeq, 554 }, 555 }, 556 } 557 } 558 559 // Select peer which has required blocks to ask missing blocks from 560 func (s *GossipStateProviderImpl) selectPeerToRequestFrom(height uint64) (*comm.RemotePeer, error) { 561 // Filter peers which posses required range of missing blocks 562 peers := s.filterPeers(s.hasRequiredHeight(height)) 563 564 n := len(peers) 565 if n == 0 { 566 return nil, errors.New("there are no peers to ask for missing blocks from") 567 } 568 569 // Select peers to ask for blocks 570 return peers[util.RandomInt(n)], nil 571 } 572 573 // filterPeers return list of peers which aligns the predicate provided 574 func (s *GossipStateProviderImpl) filterPeers(predicate func(peer discovery.NetworkMember) bool) []*comm.RemotePeer { 575 var peers []*comm.RemotePeer 576 577 for _, member := range s.gossip.PeersOfChannel(common2.ChainID(s.chainID)) { 578 if predicate(member) { 579 peers = append(peers, &comm.RemotePeer{Endpoint: member.PreferredEndpoint(), PKIID: member.PKIid}) 580 } 581 } 582 583 return peers 584 } 585 586 // hasRequiredHeight returns predicate which is capable to filter peers with ledger height above than indicated 587 // by provided input parameter 588 func (s *GossipStateProviderImpl) hasRequiredHeight(height uint64) func(peer discovery.NetworkMember) bool { 589 return func(peer discovery.NetworkMember) bool { 590 if nodeMetadata, err := FromBytes(peer.Metadata); err != nil { 591 logger.Errorf("Unable to de-serialize node meta state, error = %s", err) 592 } else if nodeMetadata.LedgerHeight >= height { 593 return true 594 } 595 596 return false 597 } 598 } 599 600 // GetBlock return ledger block given its sequence number as a parameter 601 func (s *GossipStateProviderImpl) GetBlock(index uint64) *common.Block { 602 // Try to read missing block from the ledger, should return no nil with 603 // content including at least one block 604 if blocks := s.committer.GetBlocks([]uint64{index}); blocks != nil && len(blocks) > 0 { 605 return blocks[0] 606 } 607 608 return nil 609 } 610 611 // AddPayload add new payload into state 612 func (s *GossipStateProviderImpl) AddPayload(payload *proto.Payload) error { 613 614 logger.Debug("Adding new payload into the buffer, seqNum = ", payload.SeqNum) 615 return s.payloads.Push(payload) 616 } 617 618 func (s *GossipStateProviderImpl) commitBlock(block *common.Block) error { 619 if err := s.committer.Commit(block); err != nil { 620 logger.Errorf("Got error while committing(%s)", err) 621 return err 622 } 623 624 // Update ledger level within node metadata 625 nodeMetastate := NewNodeMetastate(block.Header.Number) 626 // Decode nodeMetastate to byte array 627 b, err := nodeMetastate.Bytes() 628 if err == nil { 629 s.gossip.UpdateChannelMetadata(b, common2.ChainID(s.chainID)) 630 } else { 631 632 logger.Errorf("Unable to serialize node meta nodeMetastate, error = %s", err) 633 } 634 635 logger.Debugf("Channel [%s]: Created block [%d] with %d transaction(s)", 636 s.chainID, block.Header.Number, len(block.Data.Data)) 637 638 return nil 639 } 640 641 func min(a uint64, b uint64) uint64 { 642 return b ^ ((a ^ b) & (-(uint64(a-b) >> 63))) 643 }