github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/gossip/state/state.go (about) 1 /* 2 Copyright IBM Corp. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package state 8 9 import ( 10 "bytes" 11 "sync" 12 "sync/atomic" 13 "time" 14 15 pb "github.com/golang/protobuf/proto" 16 "github.com/hyperledger/fabric-protos-go/common" 17 proto "github.com/hyperledger/fabric-protos-go/gossip" 18 "github.com/hyperledger/fabric-protos-go/ledger/rwset" 19 "github.com/hyperledger/fabric-protos-go/peer" 20 "github.com/hyperledger/fabric-protos-go/transientstore" 21 vsccErrors "github.com/osdi23p228/fabric/common/errors" 22 "github.com/osdi23p228/fabric/gossip/api" 23 "github.com/osdi23p228/fabric/gossip/comm" 24 common2 "github.com/osdi23p228/fabric/gossip/common" 25 "github.com/osdi23p228/fabric/gossip/discovery" 26 "github.com/osdi23p228/fabric/gossip/metrics" 27 "github.com/osdi23p228/fabric/gossip/protoext" 28 "github.com/osdi23p228/fabric/gossip/util" 29 "github.com/osdi23p228/fabric/protoutil" 30 "github.com/pkg/errors" 31 ) 32 33 // GossipStateProvider is the interface to acquire sequences of the ledger blocks 34 // capable to full fill missing blocks by running state replication and 35 // sending request to get missing block to other nodes 36 type GossipStateProvider interface { 37 AddPayload(payload *proto.Payload) error 38 39 // Stop terminates state transfer object 40 Stop() 41 } 42 43 const ( 44 defAntiEntropyInterval = 10 * time.Second 45 defAntiEntropyStateResponseTimeout = 3 * time.Second 46 defAntiEntropyBatchSize = 10 47 48 defChannelBufferSize = 100 49 defAntiEntropyMaxRetries = 3 50 stragglerWarningThreshold = 100 51 52 defMaxBlockDistance = 20 53 54 blocking = true 55 nonBlocking = false 56 57 enqueueRetryInterval = time.Millisecond * 100 58 ) 59 60 // Configuration keeps state transfer configuration parameters 61 type Configuration struct { 62 AntiEntropyInterval time.Duration 63 AntiEntropyStateResponseTimeout time.Duration 64 AntiEntropyBatchSize uint64 65 MaxBlockDistance int 66 AntiEntropyMaxRetries int 67 ChannelBufferSize int 68 EnableStateTransfer bool 69 } 70 71 // GossipAdapter defines gossip/communication required interface for state provider 72 type GossipAdapter interface { 73 // Send sends a message to remote peers 74 Send(msg *proto.GossipMessage, peers ...*comm.RemotePeer) 75 76 // Accept returns a dedicated read-only channel for messages sent by other nodes that match a certain predicate. 77 // If passThrough is false, the messages are processed by the gossip layer beforehand. 78 // If passThrough is true, the gossip layer doesn't intervene and the messages 79 // can be used to send a reply back to the sender 80 Accept(acceptor common2.MessageAcceptor, passThrough bool) (<-chan *proto.GossipMessage, <-chan protoext.ReceivedMessage) 81 82 // UpdateLedgerHeight updates the ledger height the peer 83 // publishes to other peers in the channel 84 UpdateLedgerHeight(height uint64, channelID common2.ChannelID) 85 86 // PeersOfChannel returns the NetworkMembers considered alive 87 // and also subscribed to the channel given 88 PeersOfChannel(common2.ChannelID) []discovery.NetworkMember 89 } 90 91 // MCSAdapter adapter of message crypto service interface to bound 92 // specific APIs required by state transfer service 93 type MCSAdapter interface { 94 // VerifyBlock returns nil if the block is properly signed, and the claimed seqNum is the 95 // sequence number that the block's header contains. 96 // else returns error 97 VerifyBlock(channelID common2.ChannelID, seqNum uint64, signedBlock *common.Block) error 98 99 // VerifyByChannel checks that signature is a valid signature of message 100 // under a peer's verification key, but also in the context of a specific channel. 101 // If the verification succeeded, Verify returns nil meaning no error occurred. 102 // If peerIdentity is nil, then the verification fails. 103 VerifyByChannel(channelID common2.ChannelID, peerIdentity api.PeerIdentityType, signature, message []byte) error 104 } 105 106 // ledgerResources defines abilities that the ledger provides 107 type ledgerResources interface { 108 // StoreBlock deliver new block with underlined private data 109 // returns missing transaction ids 110 StoreBlock(block *common.Block, data util.PvtDataCollections) error 111 112 // StorePvtData used to persist private date into transient store 113 StorePvtData(txid string, privData *transientstore.TxPvtReadWriteSetWithConfigInfo, blckHeight uint64) error 114 115 // GetPvtDataAndBlockByNum gets block by number and also returns all related private data 116 // that requesting peer is eligible for. 117 // The order of private data in slice of PvtDataCollections doesn't imply the order of 118 // transactions in the block related to these private data, to get the correct placement 119 // need to read TxPvtData.SeqInBlock field 120 GetPvtDataAndBlockByNum(seqNum uint64, peerAuthInfo protoutil.SignedData) (*common.Block, util.PvtDataCollections, error) 121 122 // Get recent block sequence number 123 LedgerHeight() (uint64, error) 124 125 // Close ledgerResources 126 Close() 127 } 128 129 // ServicesMediator aggregated adapter to compound all mediator 130 // required by state transfer into single struct 131 type ServicesMediator struct { 132 GossipAdapter 133 MCSAdapter 134 } 135 136 // GossipStateProviderImpl the implementation of the GossipStateProvider interface 137 // the struct to handle in memory sliding window of 138 // new ledger block to be acquired by hyper ledger 139 type GossipStateProviderImpl struct { 140 logger util.Logger 141 142 // Chain id 143 chainID string 144 145 mediator *ServicesMediator 146 147 // Queue of payloads which wasn't acquired yet 148 payloads PayloadsBuffer 149 150 ledger ledgerResources 151 152 stateResponseCh chan protoext.ReceivedMessage 153 154 stateRequestCh chan protoext.ReceivedMessage 155 156 stopCh chan struct{} 157 158 once sync.Once 159 160 stateTransferActive int32 161 162 stateMetrics *metrics.StateMetrics 163 164 requestValidator *stateRequestValidator 165 166 blockingMode bool 167 168 config *StateConfig 169 } 170 171 // stateRequestValidator facilitates validation of the state request messages 172 type stateRequestValidator struct { 173 } 174 175 // validate checks for RemoteStateRequest message validity 176 func (v *stateRequestValidator) validate(request *proto.RemoteStateRequest, batchSize uint64) error { 177 if request.StartSeqNum > request.EndSeqNum { 178 return errors.Errorf("Invalid sequence interval [%d...%d).", request.StartSeqNum, request.EndSeqNum) 179 } 180 181 if request.EndSeqNum > batchSize+request.StartSeqNum { 182 return errors.Errorf("Requesting blocks range [%d-%d) greater than configured allowed"+ 183 " (%d) batching size for anti-entropy.", request.StartSeqNum, request.EndSeqNum, batchSize) 184 } 185 return nil 186 } 187 188 // NewGossipStateProvider creates state provider with coordinator instance 189 // to orchestrate arrival of private rwsets and blocks before committing them into the ledger. 190 func NewGossipStateProvider( 191 logger util.Logger, 192 chainID string, 193 services *ServicesMediator, 194 ledger ledgerResources, 195 stateMetrics *metrics.StateMetrics, 196 blockingMode bool, 197 config *StateConfig, 198 ) GossipStateProvider { 199 gossipChan, _ := services.Accept(func(message interface{}) bool { 200 // Get only data messages 201 return protoext.IsDataMsg(message.(*proto.GossipMessage)) && 202 bytes.Equal(message.(*proto.GossipMessage).Channel, []byte(chainID)) 203 }, false) 204 205 remoteStateMsgFilter := func(message interface{}) bool { 206 receivedMsg := message.(protoext.ReceivedMessage) 207 msg := receivedMsg.GetGossipMessage() 208 if !(protoext.IsRemoteStateMessage(msg.GossipMessage) || msg.GetPrivateData() != nil) { 209 return false 210 } 211 // Ensure we deal only with messages that belong to this channel 212 if !bytes.Equal(msg.Channel, []byte(chainID)) { 213 return false 214 } 215 connInfo := receivedMsg.GetConnectionInfo() 216 authErr := services.VerifyByChannel(msg.Channel, connInfo.Identity, connInfo.Auth.Signature, connInfo.Auth.SignedData) 217 if authErr != nil { 218 logger.Warning("Got unauthorized request from", string(connInfo.Identity)) 219 return false 220 } 221 return true 222 } 223 224 // Filter message which are only relevant for nodeMetastate transfer 225 _, commChan := services.Accept(remoteStateMsgFilter, true) 226 227 height, err := ledger.LedgerHeight() 228 if height == 0 { 229 // Panic here since this is an indication of invalid situation which should not happen in normal 230 // code path. 231 logger.Panic("Committer height cannot be zero, ledger should include at least one block (genesis).") 232 } 233 234 if err != nil { 235 logger.Error("Could not read ledger info to obtain current ledger height due to: ", errors.WithStack(err)) 236 // Exiting as without ledger it will be impossible 237 // to deliver new blocks 238 return nil 239 } 240 241 s := &GossipStateProviderImpl{ 242 logger: logger, 243 // MessageCryptoService 244 mediator: services, 245 // Chain ID 246 chainID: chainID, 247 // Create a queue for payloads, wrapped in a metrics buffer 248 payloads: &metricsBuffer{ 249 PayloadsBuffer: NewPayloadsBuffer(height), 250 sizeMetrics: stateMetrics.PayloadBufferSize, 251 chainID: chainID, 252 }, 253 ledger: ledger, 254 stateResponseCh: make(chan protoext.ReceivedMessage, config.StateChannelSize), 255 stateRequestCh: make(chan protoext.ReceivedMessage, config.StateChannelSize), 256 stopCh: make(chan struct{}), 257 stateTransferActive: 0, 258 once: sync.Once{}, 259 stateMetrics: stateMetrics, 260 requestValidator: &stateRequestValidator{}, 261 blockingMode: blockingMode, 262 config: config, 263 } 264 265 logger.Infof("Updating metadata information for channel %s, "+ 266 "current ledger sequence is at = %d, next expected block is = %d", chainID, height-1, s.payloads.Next()) 267 logger.Debug("Updating gossip ledger height to", height) 268 services.UpdateLedgerHeight(height, common2.ChannelID(s.chainID)) 269 270 // Listen for incoming communication 271 go s.receiveAndQueueGossipMessages(gossipChan) 272 go s.receiveAndDispatchDirectMessages(commChan) 273 // Deliver in order messages into the incoming channel 274 go s.deliverPayloads() 275 if s.config.StateEnabled { 276 // Execute anti entropy to fill missing gaps 277 go s.antiEntropy() 278 } 279 // Taking care of state request messages 280 go s.processStateRequests() 281 282 return s 283 } 284 285 func (s *GossipStateProviderImpl) receiveAndQueueGossipMessages(ch <-chan *proto.GossipMessage) { 286 for msg := range ch { 287 s.logger.Debug("Received new message via gossip channel") 288 go func(msg *proto.GossipMessage) { 289 if !bytes.Equal(msg.Channel, []byte(s.chainID)) { 290 s.logger.Warning("Received enqueue for channel", 291 string(msg.Channel), "while expecting channel", s.chainID, "ignoring enqueue") 292 return 293 } 294 295 dataMsg := msg.GetDataMsg() 296 if dataMsg != nil { 297 if err := s.addPayload(dataMsg.GetPayload(), nonBlocking); err != nil { 298 s.logger.Warningf("Block [%d] received from gossip wasn't added to payload buffer: %v", dataMsg.Payload.SeqNum, err) 299 return 300 } 301 302 } else { 303 s.logger.Debug("Gossip message received is not of data message type, usually this should not happen.") 304 } 305 }(msg) 306 } 307 } 308 309 func (s *GossipStateProviderImpl) receiveAndDispatchDirectMessages(ch <-chan protoext.ReceivedMessage) { 310 for msg := range ch { 311 s.logger.Debug("Dispatching a message", msg) 312 go func(msg protoext.ReceivedMessage) { 313 gm := msg.GetGossipMessage() 314 // Check type of the message 315 if protoext.IsRemoteStateMessage(gm.GossipMessage) { 316 s.logger.Debug("Handling direct state transfer message") 317 // Got state transfer request response 318 s.directMessage(msg) 319 } else if gm.GetPrivateData() != nil { 320 s.logger.Debug("Handling private data collection message") 321 // Handling private data replication message 322 s.privateDataMessage(msg) 323 } 324 }(msg) 325 } 326 } 327 328 func (s *GossipStateProviderImpl) privateDataMessage(msg protoext.ReceivedMessage) { 329 if !bytes.Equal(msg.GetGossipMessage().Channel, []byte(s.chainID)) { 330 s.logger.Warning("Received state transfer request for channel", 331 string(msg.GetGossipMessage().Channel), "while expecting channel", s.chainID, "skipping request...") 332 return 333 } 334 335 gossipMsg := msg.GetGossipMessage() 336 pvtDataMsg := gossipMsg.GetPrivateData() 337 338 if pvtDataMsg.Payload == nil { 339 s.logger.Warning("Malformed private data message, no payload provided") 340 return 341 } 342 343 collectionName := pvtDataMsg.Payload.CollectionName 344 txID := pvtDataMsg.Payload.TxId 345 pvtRwSet := pvtDataMsg.Payload.PrivateRwset 346 347 if len(pvtRwSet) == 0 { 348 s.logger.Warning("Malformed private data message, no rwset provided, collection name = ", collectionName) 349 return 350 } 351 352 txPvtRwSet := &rwset.TxPvtReadWriteSet{ 353 DataModel: rwset.TxReadWriteSet_KV, 354 NsPvtRwset: []*rwset.NsPvtReadWriteSet{{ 355 Namespace: pvtDataMsg.Payload.Namespace, 356 CollectionPvtRwset: []*rwset.CollectionPvtReadWriteSet{{ 357 CollectionName: collectionName, 358 Rwset: pvtRwSet, 359 }}}, 360 }, 361 } 362 363 txPvtRwSetWithConfig := &transientstore.TxPvtReadWriteSetWithConfigInfo{ 364 PvtRwset: txPvtRwSet, 365 CollectionConfigs: map[string]*peer.CollectionConfigPackage{ 366 pvtDataMsg.Payload.Namespace: pvtDataMsg.Payload.CollectionConfigs, 367 }, 368 } 369 370 if err := s.ledger.StorePvtData(txID, txPvtRwSetWithConfig, pvtDataMsg.Payload.PrivateSimHeight); err != nil { 371 s.logger.Errorf("Wasn't able to persist private data for collection %s, due to %s", collectionName, err) 372 msg.Ack(err) // Sending NACK to indicate failure of storing collection 373 } 374 375 msg.Ack(nil) 376 s.logger.Debug("Private data for collection", collectionName, "has been stored") 377 } 378 379 func (s *GossipStateProviderImpl) directMessage(msg protoext.ReceivedMessage) { 380 s.logger.Debug("[ENTER] -> directMessage") 381 defer s.logger.Debug("[EXIT] -> directMessage") 382 383 if msg == nil { 384 s.logger.Error("Got nil message via end-to-end channel, should not happen!") 385 return 386 } 387 388 if !bytes.Equal(msg.GetGossipMessage().Channel, []byte(s.chainID)) { 389 s.logger.Warning("Received state transfer request for channel", 390 string(msg.GetGossipMessage().Channel), "while expecting channel", s.chainID, "skipping request...") 391 return 392 } 393 394 incoming := msg.GetGossipMessage() 395 396 if incoming.GetStateRequest() != nil { 397 if len(s.stateRequestCh) < s.config.StateChannelSize { 398 // Forward state request to the channel, if there are too 399 // many message of state request ignore to avoid flooding. 400 s.stateRequestCh <- msg 401 } 402 } else if incoming.GetStateResponse() != nil { 403 // If no state transfer procedure activate there is 404 // no reason to process the message 405 if atomic.LoadInt32(&s.stateTransferActive) == 1 { 406 // Send signal of state response message 407 s.stateResponseCh <- msg 408 } 409 } 410 } 411 412 func (s *GossipStateProviderImpl) processStateRequests() { 413 for { 414 msg, stillOpen := <-s.stateRequestCh 415 if !stillOpen { 416 return 417 } 418 s.handleStateRequest(msg) 419 } 420 } 421 422 // handleStateRequest handles state request message, validate batch size, reads current leader state to 423 // obtain required blocks, builds response message and send it back 424 func (s *GossipStateProviderImpl) handleStateRequest(msg protoext.ReceivedMessage) { 425 if msg == nil { 426 return 427 } 428 request := msg.GetGossipMessage().GetStateRequest() 429 430 if err := s.requestValidator.validate(request, s.config.StateBatchSize); err != nil { 431 s.logger.Errorf("State request validation failed, %s. Ignoring request...", err) 432 return 433 } 434 435 currentHeight, err := s.ledger.LedgerHeight() 436 if err != nil { 437 s.logger.Errorf("Cannot access to current ledger height, due to %+v", err) 438 return 439 } 440 if currentHeight < request.EndSeqNum { 441 s.logger.Warningf("Received state request to transfer blocks with sequence numbers higher [%d...%d] "+ 442 "than available in ledger (%d)", request.StartSeqNum, request.StartSeqNum, currentHeight) 443 } 444 445 endSeqNum := min(currentHeight, request.EndSeqNum) 446 447 response := &proto.RemoteStateResponse{Payloads: make([]*proto.Payload, 0)} 448 for seqNum := request.StartSeqNum; seqNum <= endSeqNum; seqNum++ { 449 s.logger.Debug("Reading block ", seqNum, " with private data from the coordinator service") 450 connInfo := msg.GetConnectionInfo() 451 peerAuthInfo := protoutil.SignedData{ 452 Data: connInfo.Auth.SignedData, 453 Signature: connInfo.Auth.Signature, 454 Identity: connInfo.Identity, 455 } 456 block, pvtData, err := s.ledger.GetPvtDataAndBlockByNum(seqNum, peerAuthInfo) 457 458 if err != nil { 459 s.logger.Errorf("cannot read block number %d from ledger, because %+v, skipping...", seqNum, err) 460 continue 461 } 462 463 if block == nil { 464 s.logger.Errorf("Wasn't able to read block with sequence number %d from ledger, skipping....", seqNum) 465 continue 466 } 467 468 blockBytes, err := pb.Marshal(block) 469 470 if err != nil { 471 s.logger.Errorf("Could not marshal block: %+v", errors.WithStack(err)) 472 continue 473 } 474 475 var pvtBytes [][]byte 476 if pvtData != nil { 477 // Marshal private data 478 pvtBytes, err = pvtData.Marshal() 479 if err != nil { 480 s.logger.Errorf("Failed to marshal private rwset for block %d due to %+v", seqNum, errors.WithStack(err)) 481 continue 482 } 483 } 484 485 // Appending result to the response 486 response.Payloads = append(response.Payloads, &proto.Payload{ 487 SeqNum: seqNum, 488 Data: blockBytes, 489 PrivateData: pvtBytes, 490 }) 491 } 492 // Sending back response with missing blocks 493 msg.Respond(&proto.GossipMessage{ 494 // Copy nonce field from the request, so it will be possible to match response 495 Nonce: msg.GetGossipMessage().Nonce, 496 Tag: proto.GossipMessage_CHAN_OR_ORG, 497 Channel: []byte(s.chainID), 498 Content: &proto.GossipMessage_StateResponse{StateResponse: response}, 499 }) 500 } 501 502 func (s *GossipStateProviderImpl) handleStateResponse(msg protoext.ReceivedMessage) (uint64, error) { 503 max := uint64(0) 504 // Send signal that response for given nonce has been received 505 response := msg.GetGossipMessage().GetStateResponse() 506 // Extract payloads, verify and push into buffer 507 if len(response.GetPayloads()) == 0 { 508 return uint64(0), errors.New("Received state transfer response without payload") 509 } 510 for _, payload := range response.GetPayloads() { 511 s.logger.Debugf("Received payload with sequence number %d.", payload.SeqNum) 512 block, err := protoutil.UnmarshalBlock(payload.Data) 513 if err != nil { 514 s.logger.Warningf("Error unmarshaling payload to block for sequence number %d, due to %+v", payload.SeqNum, err) 515 return uint64(0), err 516 } 517 518 if err := s.mediator.VerifyBlock(common2.ChannelID(s.chainID), payload.SeqNum, block); err != nil { 519 err = errors.WithStack(err) 520 s.logger.Warningf("Error verifying block with sequence number %d, due to %+v", payload.SeqNum, err) 521 return uint64(0), err 522 } 523 if max < payload.SeqNum { 524 max = payload.SeqNum 525 } 526 527 err = s.addPayload(payload, blocking) 528 if err != nil { 529 s.logger.Warningf("Block [%d] received from block transfer wasn't added to payload buffer: %v", payload.SeqNum, err) 530 } 531 } 532 return max, nil 533 } 534 535 // Stop function sends halting signal to all go routines 536 func (s *GossipStateProviderImpl) Stop() { 537 // Make sure stop won't be executed twice 538 // and stop channel won't be used again 539 s.once.Do(func() { 540 close(s.stopCh) 541 // Close all resources 542 s.ledger.Close() 543 close(s.stateRequestCh) 544 close(s.stateResponseCh) 545 }) 546 } 547 548 func (s *GossipStateProviderImpl) deliverPayloads() { 549 for { 550 select { 551 // Wait for notification that next seq has arrived 552 case <-s.payloads.Ready(): 553 s.logger.Debugf("[%s] Ready to transfer payloads (blocks) to the ledger, next block number is = [%d]", s.chainID, s.payloads.Next()) 554 // Collect all subsequent payloads 555 for payload := s.payloads.Pop(); payload != nil; payload = s.payloads.Pop() { 556 rawBlock := &common.Block{} 557 if err := pb.Unmarshal(payload.Data, rawBlock); err != nil { 558 s.logger.Errorf("Error getting block with seqNum = %d due to (%+v)...dropping block", payload.SeqNum, errors.WithStack(err)) 559 continue 560 } 561 if rawBlock.Data == nil || rawBlock.Header == nil { 562 s.logger.Errorf("Block with claimed sequence %d has no header (%v) or data (%v)", 563 payload.SeqNum, rawBlock.Header, rawBlock.Data) 564 continue 565 } 566 s.logger.Debugf("[%s] Transferring block [%d] with %d transaction(s) to the ledger", s.chainID, payload.SeqNum, len(rawBlock.Data.Data)) 567 568 // Read all private data into slice 569 var p util.PvtDataCollections 570 if payload.PrivateData != nil { 571 err := p.Unmarshal(payload.PrivateData) 572 if err != nil { 573 s.logger.Errorf("Wasn't able to unmarshal private data for block seqNum = %d due to (%+v)...dropping block", payload.SeqNum, errors.WithStack(err)) 574 continue 575 } 576 } 577 if err := s.commitBlock(rawBlock, p); err != nil { 578 if executionErr, isExecutionErr := err.(*vsccErrors.VSCCExecutionFailureError); isExecutionErr { 579 s.logger.Errorf("Failed executing VSCC due to %v. Aborting chain processing", executionErr) 580 return 581 } 582 s.logger.Panicf("Cannot commit block to the ledger due to %+v", errors.WithStack(err)) 583 } 584 } 585 case <-s.stopCh: 586 s.logger.Debug("State provider has been stopped, finishing to push new blocks.") 587 return 588 } 589 } 590 } 591 592 func (s *GossipStateProviderImpl) antiEntropy() { 593 defer s.logger.Debug("State Provider stopped, stopping anti entropy procedure.") 594 595 for { 596 select { 597 case <-s.stopCh: 598 return 599 case <-time.After(s.config.StateCheckInterval): 600 ourHeight, err := s.ledger.LedgerHeight() 601 if err != nil { 602 // Unable to read from ledger continue to the next round 603 s.logger.Errorf("Cannot obtain ledger height, due to %+v", errors.WithStack(err)) 604 continue 605 } 606 if ourHeight == 0 { 607 s.logger.Error("Ledger reported block height of 0 but this should be impossible") 608 continue 609 } 610 maxHeight := s.maxAvailableLedgerHeight() 611 if ourHeight >= maxHeight { 612 continue 613 } 614 615 s.requestBlocksInRange(uint64(ourHeight), uint64(maxHeight)-1) 616 } 617 } 618 } 619 620 // maxAvailableLedgerHeight iterates over all available peers and checks advertised meta state to 621 // find maximum available ledger height across peers 622 func (s *GossipStateProviderImpl) maxAvailableLedgerHeight() uint64 { 623 max := uint64(0) 624 for _, p := range s.mediator.PeersOfChannel(common2.ChannelID(s.chainID)) { 625 if p.Properties == nil { 626 s.logger.Debug("Peer", p.PreferredEndpoint(), "doesn't have properties, skipping it") 627 continue 628 } 629 peerHeight := p.Properties.LedgerHeight 630 if max < peerHeight { 631 max = peerHeight 632 } 633 } 634 return max 635 } 636 637 // requestBlocksInRange capable to acquire blocks with sequence 638 // numbers in the range [start...end). 639 func (s *GossipStateProviderImpl) requestBlocksInRange(start uint64, end uint64) { 640 atomic.StoreInt32(&s.stateTransferActive, 1) 641 defer atomic.StoreInt32(&s.stateTransferActive, 0) 642 643 for prev := start; prev <= end; { 644 next := min(end, prev+s.config.StateBatchSize) 645 646 gossipMsg := s.stateRequestMessage(prev, next) 647 648 responseReceived := false 649 tryCounts := 0 650 651 for !responseReceived { 652 if tryCounts > s.config.StateMaxRetries { 653 s.logger.Warningf("Wasn't able to get blocks in range [%d...%d), after %d retries", 654 prev, next, tryCounts) 655 return 656 } 657 // Select peers to ask for blocks 658 peer, err := s.selectPeerToRequestFrom(next) 659 if err != nil { 660 s.logger.Warningf("Cannot send state request for blocks in range [%d...%d), due to %+v", 661 prev, next, errors.WithStack(err)) 662 return 663 } 664 665 s.logger.Debugf("State transfer, with peer %s, requesting blocks in range [%d...%d), "+ 666 "for chainID %s", peer.Endpoint, prev, next, s.chainID) 667 668 s.mediator.Send(gossipMsg, peer) 669 tryCounts++ 670 671 // Wait until timeout or response arrival 672 select { 673 case msg, stillOpen := <-s.stateResponseCh: 674 if !stillOpen { 675 return 676 } 677 if msg.GetGossipMessage().Nonce != 678 gossipMsg.Nonce { 679 continue 680 } 681 // Got corresponding response for state request, can continue 682 index, err := s.handleStateResponse(msg) 683 if err != nil { 684 s.logger.Warningf("Wasn't able to process state response for "+ 685 "blocks [%d...%d], due to %+v", prev, next, errors.WithStack(err)) 686 continue 687 } 688 prev = index + 1 689 responseReceived = true 690 case <-time.After(s.config.StateResponseTimeout): 691 } 692 } 693 } 694 } 695 696 // stateRequestMessage generates state request message for given blocks in range [beginSeq...endSeq] 697 func (s *GossipStateProviderImpl) stateRequestMessage(beginSeq uint64, endSeq uint64) *proto.GossipMessage { 698 return &proto.GossipMessage{ 699 Nonce: util.RandomUInt64(), 700 Tag: proto.GossipMessage_CHAN_OR_ORG, 701 Channel: []byte(s.chainID), 702 Content: &proto.GossipMessage_StateRequest{ 703 StateRequest: &proto.RemoteStateRequest{ 704 StartSeqNum: beginSeq, 705 EndSeqNum: endSeq, 706 }, 707 }, 708 } 709 } 710 711 // selectPeerToRequestFrom selects peer which has required blocks to ask missing blocks from 712 func (s *GossipStateProviderImpl) selectPeerToRequestFrom(height uint64) (*comm.RemotePeer, error) { 713 // Filter peers which posses required range of missing blocks 714 peers := s.filterPeers(s.hasRequiredHeight(height)) 715 716 n := len(peers) 717 if n == 0 { 718 return nil, errors.New("there are no peers to ask for missing blocks from") 719 } 720 721 // Select peer to ask for blocks 722 return peers[util.RandomInt(n)], nil 723 } 724 725 // filterPeers returns list of peers which aligns the predicate provided 726 func (s *GossipStateProviderImpl) filterPeers(predicate func(peer discovery.NetworkMember) bool) []*comm.RemotePeer { 727 var peers []*comm.RemotePeer 728 729 for _, member := range s.mediator.PeersOfChannel(common2.ChannelID(s.chainID)) { 730 if predicate(member) { 731 peers = append(peers, &comm.RemotePeer{Endpoint: member.PreferredEndpoint(), PKIID: member.PKIid}) 732 } 733 } 734 735 return peers 736 } 737 738 // hasRequiredHeight returns predicate which is capable to filter peers with ledger height above than indicated 739 // by provided input parameter 740 func (s *GossipStateProviderImpl) hasRequiredHeight(height uint64) func(peer discovery.NetworkMember) bool { 741 return func(peer discovery.NetworkMember) bool { 742 if peer.Properties != nil { 743 return peer.Properties.LedgerHeight >= height 744 } 745 s.logger.Debug(peer.PreferredEndpoint(), "doesn't have properties") 746 return false 747 } 748 } 749 750 // AddPayload adds new payload into state. 751 func (s *GossipStateProviderImpl) AddPayload(payload *proto.Payload) error { 752 return s.addPayload(payload, s.blockingMode) 753 } 754 755 // addPayload adds new payload into state. It may (or may not) block according to the 756 // given parameter. If it gets a block while in blocking mode - it would wait until 757 // the block is sent into the payloads buffer. 758 // Else - it may drop the block, if the payload buffer is too full. 759 func (s *GossipStateProviderImpl) addPayload(payload *proto.Payload, blockingMode bool) error { 760 if payload == nil { 761 return errors.New("Given payload is nil") 762 } 763 s.logger.Debugf("[%s] Adding payload to local buffer, blockNum = [%d]", s.chainID, payload.SeqNum) 764 height, err := s.ledger.LedgerHeight() 765 if err != nil { 766 return errors.Wrap(err, "Failed obtaining ledger height") 767 } 768 769 if !blockingMode && payload.SeqNum-height >= uint64(s.config.StateBlockBufferSize) { 770 if s.straggler(height, payload) { 771 s.logger.Warningf("[%s] Current block height (%d) is too far behind other peers at height (%d) to be able to receive blocks "+ 772 "without state transfer which is disabled in the configuration "+ 773 "(peer.gossip.state.enabled = false). Consider enabling it or setting the peer explicitly to be a leader (peer.gossip.orgLeader = true) "+ 774 "in order to pull blocks directly from the ordering service.", 775 s.chainID, height, payload.SeqNum+1) 776 } 777 return errors.Errorf("Ledger height is at %d, cannot enqueue block with sequence of %d", height, payload.SeqNum) 778 } 779 780 for blockingMode && s.payloads.Size() > s.config.StateBlockBufferSize*2 { 781 time.Sleep(enqueueRetryInterval) 782 } 783 784 s.payloads.Push(payload) 785 s.logger.Debugf("Blocks payloads buffer size for channel [%s] is %d blocks", s.chainID, s.payloads.Size()) 786 return nil 787 } 788 789 func (s *GossipStateProviderImpl) straggler(currHeight uint64, receivedPayload *proto.Payload) bool { 790 // If state transfer is disabled, there is no way to request blocks from peers that their ledger has advanced too far. 791 stateDisabled := !s.config.StateEnabled 792 // We are too far behind if we received a block with a sequence number more than stragglerWarningThreshold ahead of our height. 793 tooFarBehind := currHeight+stragglerWarningThreshold < receivedPayload.SeqNum 794 // We depend on other peers for blocks if we use leader election, or we are not explicitly configured to be an org leader. 795 peerDependent := s.config.UseLeaderElection || !s.config.OrgLeader 796 return stateDisabled && tooFarBehind && peerDependent 797 } 798 799 func (s *GossipStateProviderImpl) commitBlock(block *common.Block, pvtData util.PvtDataCollections) error { 800 801 t1 := time.Now() 802 803 // Commit block with available private transactions 804 if err := s.ledger.StoreBlock(block, pvtData); err != nil { 805 s.logger.Errorf("Got error while committing(%+v)", errors.WithStack(err)) 806 return err 807 } 808 809 sinceT1 := time.Since(t1) 810 s.stateMetrics.CommitDuration.With("channel", s.chainID).Observe(sinceT1.Seconds()) 811 812 // Update ledger height 813 s.mediator.UpdateLedgerHeight(block.Header.Number+1, common2.ChannelID(s.chainID)) 814 s.logger.Debugf("[%s] Committed block [%d] with %d transaction(s)", 815 s.chainID, block.Header.Number, len(block.Data.Data)) 816 817 s.stateMetrics.Height.With("channel", s.chainID).Set(float64(block.Header.Number + 1)) 818 819 return nil 820 } 821 822 func min(a uint64, b uint64) uint64 { 823 return b ^ ((a ^ b) & (-(uint64(a-b) >> 63))) 824 }