github.com/yimialmonte/fabric@v2.1.1+incompatible/gossip/privdata/distributor.go (about) 1 /* 2 Copyright IBM Corp. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package privdata 8 9 import ( 10 "bytes" 11 "fmt" 12 "math/rand" 13 "sync" 14 "sync/atomic" 15 "time" 16 17 "github.com/golang/protobuf/proto" 18 protosgossip "github.com/hyperledger/fabric-protos-go/gossip" 19 "github.com/hyperledger/fabric-protos-go/ledger/rwset" 20 "github.com/hyperledger/fabric-protos-go/peer" 21 "github.com/hyperledger/fabric-protos-go/transientstore" 22 "github.com/hyperledger/fabric/core/common/privdata" 23 "github.com/hyperledger/fabric/gossip/api" 24 gossipCommon "github.com/hyperledger/fabric/gossip/common" 25 "github.com/hyperledger/fabric/gossip/discovery" 26 "github.com/hyperledger/fabric/gossip/filter" 27 gossipgossip "github.com/hyperledger/fabric/gossip/gossip" 28 "github.com/hyperledger/fabric/gossip/metrics" 29 "github.com/hyperledger/fabric/gossip/protoext" 30 "github.com/hyperledger/fabric/gossip/util" 31 "github.com/hyperledger/fabric/msp" 32 "github.com/hyperledger/fabric/protoutil" 33 "github.com/pkg/errors" 34 ) 35 36 // gossipAdapter an adapter for API's required from gossip module 37 type gossipAdapter interface { 38 // SendByCriteria sends a given message to all peers that match the given SendCriteria 39 SendByCriteria(message *protoext.SignedGossipMessage, criteria gossipgossip.SendCriteria) error 40 41 // PeerFilter receives a SubChannelSelectionCriteria and returns a RoutingFilter that selects 42 // only peer identities that match the given criteria, and that they published their channel participation 43 PeerFilter(channel gossipCommon.ChannelID, messagePredicate api.SubChannelSelectionCriteria) (filter.RoutingFilter, error) 44 45 // IdentityInfo returns information known peer identities 46 IdentityInfo() api.PeerIdentitySet 47 48 // PeersOfChannel returns the NetworkMembers considered alive 49 // and also subscribed to the channel given 50 PeersOfChannel(gossipCommon.ChannelID) []discovery.NetworkMember 51 } 52 53 // PvtDataDistributor interface to defines API of distributing private data 54 type PvtDataDistributor interface { 55 // Distribute broadcast reliably private data read write set based on policies 56 Distribute(txID string, privData *transientstore.TxPvtReadWriteSetWithConfigInfo, blkHt uint64) error 57 } 58 59 // IdentityDeserializerFactory is a factory interface to create 60 // IdentityDeserializer for given channel 61 type IdentityDeserializerFactory interface { 62 // GetIdentityDeserializer returns an IdentityDeserializer 63 // instance for the specified chain 64 GetIdentityDeserializer(chainID string) msp.IdentityDeserializer 65 } 66 67 // IdentityDeserializerFactoryFunc is a function adapter for 68 // IdentityDeserializerFactory. 69 type IdentityDeserializerFactoryFunc func(chainID string) msp.IdentityDeserializer 70 71 func (i IdentityDeserializerFactoryFunc) GetIdentityDeserializer(chainID string) msp.IdentityDeserializer { 72 return i(chainID) 73 } 74 75 // distributorImpl the implementation of the private data distributor interface 76 type distributorImpl struct { 77 chainID string 78 gossipAdapter 79 CollectionAccessFactory 80 pushAckTimeout time.Duration 81 metrics *metrics.PrivdataMetrics 82 } 83 84 //go:generate mockery -dir . -name CollectionAccessFactory -case underscore -output ./mocks/ 85 86 // CollectionAccessFactory an interface to generate collection access policy 87 type CollectionAccessFactory interface { 88 // AccessPolicy based on collection configuration 89 AccessPolicy(config *peer.CollectionConfig, chainID string) (privdata.CollectionAccessPolicy, error) 90 } 91 92 //go:generate mockery -dir . -name CollectionAccessPolicy -case underscore -output ./mocks/ 93 94 type CollectionAccessPolicy interface { 95 privdata.CollectionAccessPolicy 96 } 97 98 // policyAccessFactory the implementation of CollectionAccessFactory 99 type policyAccessFactory struct { 100 IdentityDeserializerFactory 101 } 102 103 func (p *policyAccessFactory) AccessPolicy(config *peer.CollectionConfig, chainID string) (privdata.CollectionAccessPolicy, error) { 104 colAP := &privdata.SimpleCollection{} 105 switch cconf := config.Payload.(type) { 106 case *peer.CollectionConfig_StaticCollectionConfig: 107 err := colAP.Setup(cconf.StaticCollectionConfig, p.GetIdentityDeserializer(chainID)) 108 if err != nil { 109 return nil, errors.WithMessagef(err, "error setting up collection %#v", cconf.StaticCollectionConfig.Name) 110 } 111 default: 112 return nil, errors.New("unexpected collection type") 113 } 114 return colAP, nil 115 } 116 117 // NewCollectionAccessFactory 118 func NewCollectionAccessFactory(factory IdentityDeserializerFactory) CollectionAccessFactory { 119 return &policyAccessFactory{ 120 IdentityDeserializerFactory: factory, 121 } 122 } 123 124 // NewDistributor a constructor for private data distributor capable to send 125 // private read write sets for underlying collection 126 func NewDistributor(chainID string, gossip gossipAdapter, factory CollectionAccessFactory, 127 metrics *metrics.PrivdataMetrics, pushAckTimeout time.Duration) PvtDataDistributor { 128 return &distributorImpl{ 129 chainID: chainID, 130 gossipAdapter: gossip, 131 CollectionAccessFactory: factory, 132 pushAckTimeout: pushAckTimeout, 133 metrics: metrics, 134 } 135 } 136 137 // Distribute broadcast reliably private data read write set based on policies 138 func (d *distributorImpl) Distribute(txID string, privData *transientstore.TxPvtReadWriteSetWithConfigInfo, blkHt uint64) error { 139 disseminationPlan, err := d.computeDisseminationPlan(txID, privData, blkHt) 140 if err != nil { 141 return errors.WithStack(err) 142 } 143 return d.disseminate(disseminationPlan) 144 } 145 146 type dissemination struct { 147 msg *protoext.SignedGossipMessage 148 criteria gossipgossip.SendCriteria 149 } 150 151 func (d *distributorImpl) computeDisseminationPlan(txID string, 152 privDataWithConfig *transientstore.TxPvtReadWriteSetWithConfigInfo, 153 blkHt uint64) ([]*dissemination, error) { 154 privData := privDataWithConfig.PvtRwset 155 var disseminationPlan []*dissemination 156 for _, pvtRwset := range privData.NsPvtRwset { 157 namespace := pvtRwset.Namespace 158 configPackage, found := privDataWithConfig.CollectionConfigs[namespace] 159 if !found { 160 logger.Error("Collection config package for", namespace, "chaincode is not provided") 161 return nil, errors.New(fmt.Sprint("collection config package for", namespace, "chaincode is not provided")) 162 } 163 164 for _, collection := range pvtRwset.CollectionPvtRwset { 165 colCP, err := d.getCollectionConfig(configPackage, collection) 166 collectionName := collection.CollectionName 167 if err != nil { 168 logger.Error("Could not find collection access policy for", namespace, " and collection", collectionName, "error", err) 169 return nil, errors.WithMessage(err, fmt.Sprint("could not find collection access policy for", namespace, " and collection", collectionName, "error", err)) 170 } 171 172 colAP, err := d.AccessPolicy(colCP, d.chainID) 173 if err != nil { 174 logger.Error("Could not obtain collection access policy, collection name", collectionName, "due to", err) 175 return nil, errors.Wrap(err, fmt.Sprint("Could not obtain collection access policy, collection name", collectionName, "due to", err)) 176 } 177 178 colFilter := colAP.AccessFilter() 179 if colFilter == nil { 180 logger.Error("Collection access policy for", collectionName, "has no filter") 181 return nil, errors.Errorf("No collection access policy filter computed for %v", collectionName) 182 } 183 184 pvtDataMsg, err := d.createPrivateDataMessage(txID, namespace, collection, &peer.CollectionConfigPackage{Config: []*peer.CollectionConfig{colCP}}, blkHt) 185 if err != nil { 186 return nil, errors.WithStack(err) 187 } 188 189 logger.Debugf("Computing dissemination plan for collection [%s]", collectionName) 190 dPlan, err := d.disseminationPlanForMsg(colAP, colFilter, pvtDataMsg) 191 if err != nil { 192 return nil, errors.WithMessagef(err, "could not build private data dissemination plan for chaincode %s and collection %s", namespace, collectionName) 193 } 194 disseminationPlan = append(disseminationPlan, dPlan...) 195 } 196 } 197 return disseminationPlan, nil 198 } 199 200 func (d *distributorImpl) getCollectionConfig(config *peer.CollectionConfigPackage, collection *rwset.CollectionPvtReadWriteSet) (*peer.CollectionConfig, error) { 201 for _, c := range config.Config { 202 if staticConfig := c.GetStaticCollectionConfig(); staticConfig != nil { 203 if staticConfig.Name == collection.CollectionName { 204 return c, nil 205 } 206 } 207 } 208 return nil, errors.New(fmt.Sprint("no configuration for collection", collection.CollectionName, "found")) 209 } 210 211 func (d *distributorImpl) disseminationPlanForMsg(colAP privdata.CollectionAccessPolicy, colFilter privdata.Filter, pvtDataMsg *protoext.SignedGossipMessage) ([]*dissemination, error) { 212 var disseminationPlan []*dissemination 213 214 routingFilter, err := d.gossipAdapter.PeerFilter(gossipCommon.ChannelID(d.chainID), func(signature api.PeerSignature) bool { 215 return colFilter(protoutil.SignedData{ 216 Data: signature.Message, 217 Signature: signature.Signature, 218 Identity: []byte(signature.PeerIdentity), 219 }) 220 }) 221 222 if err != nil { 223 logger.Error("Failed to retrieve peer routing filter for channel", d.chainID, ":", err) 224 return nil, err 225 } 226 227 m := pvtDataMsg.GetPrivateData().Payload 228 229 eligiblePeers := d.eligiblePeersOfChannel(routingFilter) 230 231 // With the shift to per peer dissemination in FAB-15389, we must first check 232 // that there are enough eligible peers to satisfy RequiredPeerCount. 233 if (len(eligiblePeers)) < colAP.RequiredPeerCount() { 234 return nil, errors.Errorf("required to disseminate to at least %d peers, but know of only %d eligible peers", colAP.RequiredPeerCount(), len(eligiblePeers)) 235 } 236 237 // Group eligible peers by org so that we can disseminate across orgs first 238 identitySetsByOrg := d.identitiesOfEligiblePeersByOrg(eligiblePeers, colAP) 239 240 // peerEndpoints are used for dissemination debug only 241 peerEndpoints := map[string]string{} 242 for _, peer := range eligiblePeers { 243 epToAdd := peer.Endpoint 244 if epToAdd == "" { 245 epToAdd = peer.InternalEndpoint 246 } 247 peerEndpoints[string(peer.PKIid)] = epToAdd 248 } 249 250 // Initialize maximumPeerRemainingCount and requiredPeerRemainingCount, 251 // these will be decremented until we've selected enough peers for dissemination 252 maximumPeerRemainingCount := colAP.MaximumPeerCount() 253 requiredPeerRemainingCount := colAP.RequiredPeerCount() 254 255 remainingPeersAcrossOrgs := []api.PeerIdentityInfo{} 256 selectedPeerEndpointsForDebug := []string{} 257 258 rand.Seed(time.Now().Unix()) 259 260 // PHASE 1 - Select one peer from each eligible org 261 if maximumPeerRemainingCount > 0 { 262 for _, selectionPeersForOrg := range identitySetsByOrg { 263 264 // Peers are tagged as a required peer (acksRequired=1) for RequiredPeerCount up front before dissemination. 265 // TODO It would be better to attempt dissemination to MaxPeerCount first, and then verify that enough sends were acknowledged to meet RequiredPeerCount. 266 acksRequired := 1 267 if requiredPeerRemainingCount == 0 { 268 acksRequired = 0 269 } 270 271 selectedPeerIndex := rand.Intn(len(selectionPeersForOrg)) 272 peer2SendPerOrg := selectionPeersForOrg[selectedPeerIndex] 273 selectedPeerEndpointsForDebug = append(selectedPeerEndpointsForDebug, peerEndpoints[string(peer2SendPerOrg.PKIId)]) 274 sc := gossipgossip.SendCriteria{ 275 Timeout: d.pushAckTimeout, 276 Channel: gossipCommon.ChannelID(d.chainID), 277 MaxPeers: 1, 278 MinAck: acksRequired, 279 IsEligible: func(member discovery.NetworkMember) bool { 280 return bytes.Equal(member.PKIid, peer2SendPerOrg.PKIId) 281 }, 282 } 283 disseminationPlan = append(disseminationPlan, &dissemination{ 284 criteria: sc, 285 msg: &protoext.SignedGossipMessage{ 286 Envelope: proto.Clone(pvtDataMsg.Envelope).(*protosgossip.Envelope), 287 GossipMessage: proto.Clone(pvtDataMsg.GossipMessage).(*protosgossip.GossipMessage), 288 }, 289 }) 290 291 // Add unselected peers to remainingPeersAcrossOrgs 292 for i, peer := range selectionPeersForOrg { 293 if i != selectedPeerIndex { 294 remainingPeersAcrossOrgs = append(remainingPeersAcrossOrgs, peer) 295 } 296 } 297 298 if requiredPeerRemainingCount > 0 { 299 requiredPeerRemainingCount-- 300 } 301 302 maximumPeerRemainingCount-- 303 if maximumPeerRemainingCount == 0 { 304 logger.Debug("MaximumPeerCount satisfied") 305 logger.Debugf("Disseminating private RWSet for TxID [%s] namespace [%s] collection [%s] to peers: %v", m.TxId, m.Namespace, m.CollectionName, selectedPeerEndpointsForDebug) 306 return disseminationPlan, nil 307 } 308 } 309 } 310 311 // PHASE 2 - Select additional peers to satisfy colAP.MaximumPeerCount() if there are still peers in the remainingPeersAcrossOrgs pool 312 numRemainingPeersToSelect := maximumPeerRemainingCount 313 if len(remainingPeersAcrossOrgs) < maximumPeerRemainingCount { 314 numRemainingPeersToSelect = len(remainingPeersAcrossOrgs) 315 } 316 if numRemainingPeersToSelect > 0 { 317 logger.Debugf("MaximumPeerCount not yet satisfied after picking one peer per org, selecting %d more peer(s) for dissemination", numRemainingPeersToSelect) 318 } 319 for maximumPeerRemainingCount > 0 && len(remainingPeersAcrossOrgs) > 0 { 320 required := 1 321 if requiredPeerRemainingCount == 0 { 322 required = 0 323 } 324 selectedPeerIndex := rand.Intn(len(remainingPeersAcrossOrgs)) 325 peer2Send := remainingPeersAcrossOrgs[selectedPeerIndex] 326 selectedPeerEndpointsForDebug = append(selectedPeerEndpointsForDebug, peerEndpoints[string(peer2Send.PKIId)]) 327 sc := gossipgossip.SendCriteria{ 328 Timeout: d.pushAckTimeout, 329 Channel: gossipCommon.ChannelID(d.chainID), 330 MaxPeers: 1, 331 MinAck: required, 332 IsEligible: func(member discovery.NetworkMember) bool { 333 return bytes.Equal(member.PKIid, peer2Send.PKIId) 334 }, 335 } 336 disseminationPlan = append(disseminationPlan, &dissemination{ 337 criteria: sc, 338 msg: &protoext.SignedGossipMessage{ 339 Envelope: proto.Clone(pvtDataMsg.Envelope).(*protosgossip.Envelope), 340 GossipMessage: proto.Clone(pvtDataMsg.GossipMessage).(*protosgossip.GossipMessage), 341 }, 342 }) 343 if requiredPeerRemainingCount > 0 { 344 requiredPeerRemainingCount-- 345 } 346 347 maximumPeerRemainingCount-- 348 349 // remove the selected peer from remaining peers 350 remainingPeersAcrossOrgs = append(remainingPeersAcrossOrgs[:selectedPeerIndex], remainingPeersAcrossOrgs[selectedPeerIndex+1:]...) 351 } 352 353 logger.Debugf("Disseminating private RWSet for TxID [%s] namespace [%s] collection [%s] to peers: %v", m.TxId, m.Namespace, m.CollectionName, selectedPeerEndpointsForDebug) 354 return disseminationPlan, nil 355 } 356 357 // identitiesOfEligiblePeersByOrg returns the peers eligible for a collection (aka PeerIdentitySet) grouped in a hash map keyed by orgid 358 func (d *distributorImpl) identitiesOfEligiblePeersByOrg(eligiblePeers []discovery.NetworkMember, colAP privdata.CollectionAccessPolicy) map[string]api.PeerIdentitySet { 359 return d.gossipAdapter.IdentityInfo(). 360 Filter(func(info api.PeerIdentityInfo) bool { 361 if _, ok := colAP.MemberOrgs()[string(info.Organization)]; ok { 362 return true 363 } 364 // peer not in the org 365 return false 366 }).Filter(func(info api.PeerIdentityInfo) bool { 367 for _, peer := range eligiblePeers { 368 if bytes.Equal(info.PKIId, peer.PKIid) { 369 return true 370 } 371 } 372 // peer not in the channel 373 return false 374 }).ByOrg() 375 } 376 377 func (d *distributorImpl) eligiblePeersOfChannel(routingFilter filter.RoutingFilter) []discovery.NetworkMember { 378 var eligiblePeers []discovery.NetworkMember 379 for _, peer := range d.gossipAdapter.PeersOfChannel(gossipCommon.ChannelID(d.chainID)) { 380 if routingFilter(peer) { 381 eligiblePeers = append(eligiblePeers, peer) 382 } 383 } 384 return eligiblePeers 385 } 386 387 func (d *distributorImpl) disseminate(disseminationPlan []*dissemination) error { 388 var failures uint32 389 var wg sync.WaitGroup 390 wg.Add(len(disseminationPlan)) 391 start := time.Now() 392 for _, dis := range disseminationPlan { 393 go func(dis *dissemination) { 394 defer wg.Done() 395 defer d.reportSendDuration(start) 396 err := d.SendByCriteria(dis.msg, dis.criteria) 397 if err != nil { 398 atomic.AddUint32(&failures, 1) 399 m := dis.msg.GetPrivateData().Payload 400 logger.Error("Failed disseminating private RWSet for TxID", m.TxId, ", namespace", m.Namespace, "collection", m.CollectionName, ":", err) 401 } 402 }(dis) 403 } 404 wg.Wait() 405 failureCount := atomic.LoadUint32(&failures) 406 if failureCount != 0 { 407 return errors.Errorf("Failed disseminating %d out of %d private dissemination plans", failureCount, len(disseminationPlan)) 408 } 409 return nil 410 } 411 412 func (d *distributorImpl) reportSendDuration(startTime time.Time) { 413 d.metrics.SendDuration.With("channel", d.chainID).Observe(time.Since(startTime).Seconds()) 414 } 415 416 func (d *distributorImpl) createPrivateDataMessage(txID, namespace string, 417 collection *rwset.CollectionPvtReadWriteSet, 418 ccp *peer.CollectionConfigPackage, 419 blkHt uint64) (*protoext.SignedGossipMessage, error) { 420 msg := &protosgossip.GossipMessage{ 421 Channel: []byte(d.chainID), 422 Nonce: util.RandomUInt64(), 423 Tag: protosgossip.GossipMessage_CHAN_ONLY, 424 Content: &protosgossip.GossipMessage_PrivateData{ 425 PrivateData: &protosgossip.PrivateDataMessage{ 426 Payload: &protosgossip.PrivatePayload{ 427 Namespace: namespace, 428 CollectionName: collection.CollectionName, 429 TxId: txID, 430 PrivateRwset: collection.Rwset, 431 PrivateSimHeight: blkHt, 432 CollectionConfigs: ccp, 433 }, 434 }, 435 }, 436 } 437 438 pvtDataMsg, err := protoext.NoopSign(msg) 439 if err != nil { 440 return nil, err 441 } 442 return pvtDataMsg, nil 443 }