github.com/tenywen/fabric@v1.0.0-beta.0.20170620030522-a5b1ed380643/gossip/election/election.go (about) 1 /* 2 Copyright IBM Corp. 2016 All Rights Reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package election 18 19 import ( 20 "bytes" 21 "sync" 22 "sync/atomic" 23 "time" 24 25 "github.com/hyperledger/fabric/gossip/util" 26 "github.com/op/go-logging" 27 "github.com/spf13/viper" 28 ) 29 30 // Gossip leader election module 31 // Algorithm properties: 32 // - Peers break symmetry by comparing IDs 33 // - Each peer is either a leader or a follower, 34 // and the aim is to have exactly 1 leader if the membership view 35 // is the same for all peers 36 // - If the network is partitioned into 2 or more sets, the number of leaders 37 // is the number of network partitions, but when the partition heals, 38 // only 1 leader should be left eventually 39 // - Peers communicate by gossiping leadership proposal or declaration messages 40 41 // The Algorithm, in pseudo code: 42 // 43 // 44 // variables: 45 // leaderKnown = false 46 // 47 // Invariant: 48 // Peer listens for messages from remote peers 49 // and whenever it receives a leadership declaration, 50 // leaderKnown is set to true 51 // 52 // Startup(): 53 // wait for membership view to stabilize, or for a leadership declaration is received 54 // or the startup timeout expires. 55 // goto SteadyState() 56 // 57 // SteadyState(): 58 // while true: 59 // If leaderKnown is false: 60 // LeaderElection() 61 // If you are the leader: 62 // Broadcast leadership declaration 63 // If a leadership declaration was received from 64 // a peer with a lower ID, 65 // become a follower 66 // Else, you're a follower: 67 // If haven't received a leadership declaration within 68 // a time threshold: 69 // set leaderKnown to false 70 // 71 // LeaderElection(): 72 // Gossip leadership proposal message 73 // Collect messages from other peers sent within a time period 74 // If received a leadership declaration: 75 // return 76 // Iterate over all proposal messages collected. 77 // If a proposal message from a peer with an ID lower 78 // than yourself was received, return. 79 // Else, declare yourself a leader 80 81 // LeaderElectionAdapter is used by the leader election module 82 // to send and receive messages and to get membership information 83 type LeaderElectionAdapter interface { 84 // Gossip gossips a message to other peers 85 Gossip(Msg) 86 87 // Accept returns a channel that emits messages 88 Accept() <-chan Msg 89 90 // CreateProposalMessage 91 CreateMessage(isDeclaration bool) Msg 92 93 // Peers returns a list of peers considered alive 94 Peers() []Peer 95 } 96 97 type leadershipCallback func(isLeader bool) 98 99 // LeaderElectionService is the object that runs the leader election algorithm 100 type LeaderElectionService interface { 101 // IsLeader returns whether this peer is a leader or not 102 IsLeader() bool 103 104 // Stop stops the LeaderElectionService 105 Stop() 106 } 107 108 type peerID []byte 109 110 // Peer describes a remote peer 111 type Peer interface { 112 // ID returns the ID of the peer 113 ID() peerID 114 } 115 116 // Msg describes a message sent from a remote peer 117 type Msg interface { 118 // SenderID returns the ID of the peer sent the message 119 SenderID() peerID 120 // IsProposal returns whether this message is a leadership proposal 121 IsProposal() bool 122 // IsDeclaration returns whether this message is a leadership declaration 123 IsDeclaration() bool 124 } 125 126 func noopCallback(_ bool) { 127 } 128 129 // NewLeaderElectionService returns a new LeaderElectionService 130 func NewLeaderElectionService(adapter LeaderElectionAdapter, id string, callback leadershipCallback) LeaderElectionService { 131 if len(id) == 0 { 132 panic("Empty id") 133 } 134 le := &leaderElectionSvcImpl{ 135 id: peerID(id), 136 proposals: util.NewSet(), 137 adapter: adapter, 138 stopChan: make(chan struct{}, 1), 139 interruptChan: make(chan struct{}, 1), 140 logger: util.GetLogger(util.LoggingElectionModule, ""), 141 callback: noopCallback, 142 } 143 144 if callback != nil { 145 le.callback = callback 146 } 147 148 go le.start() 149 return le 150 } 151 152 // leaderElectionSvcImpl is an implementation of a LeaderElectionService 153 type leaderElectionSvcImpl struct { 154 id peerID 155 proposals *util.Set 156 sync.Mutex 157 stopChan chan struct{} 158 interruptChan chan struct{} 159 stopWG sync.WaitGroup 160 isLeader int32 161 toDie int32 162 leaderExists int32 163 sleeping bool 164 adapter LeaderElectionAdapter 165 logger *logging.Logger 166 callback leadershipCallback 167 } 168 169 func (le *leaderElectionSvcImpl) start() { 170 le.stopWG.Add(2) 171 go le.handleMessages() 172 le.waitForMembershipStabilization(getStartupGracePeriod()) 173 go le.run() 174 } 175 176 func (le *leaderElectionSvcImpl) handleMessages() { 177 le.logger.Debug(le.id, ": Entering") 178 defer le.logger.Debug(le.id, ": Exiting") 179 defer le.stopWG.Done() 180 msgChan := le.adapter.Accept() 181 for { 182 select { 183 case <-le.stopChan: 184 le.stopChan <- struct{}{} 185 return 186 case msg := <-msgChan: 187 if !le.isAlive(msg.SenderID()) { 188 le.logger.Debug(le.id, ": Got message from", msg.SenderID(), "but it is not in the view") 189 break 190 } 191 le.handleMessage(msg) 192 } 193 } 194 } 195 196 func (le *leaderElectionSvcImpl) handleMessage(msg Msg) { 197 msgType := "proposal" 198 if msg.IsDeclaration() { 199 msgType = "declaration" 200 } 201 le.logger.Debug(le.id, ":", msg.SenderID(), "sent us", msgType) 202 le.Lock() 203 defer le.Unlock() 204 205 if msg.IsProposal() { 206 le.proposals.Add(string(msg.SenderID())) 207 } else if msg.IsDeclaration() { 208 atomic.StoreInt32(&le.leaderExists, int32(1)) 209 if le.sleeping && len(le.interruptChan) == 0 { 210 le.interruptChan <- struct{}{} 211 } 212 if bytes.Compare(msg.SenderID(), le.id) < 0 && le.IsLeader() { 213 le.stopBeingLeader() 214 } 215 } else { 216 // We shouldn't get here 217 le.logger.Error("Got a message that's not a proposal and not a declaration") 218 } 219 } 220 221 // waitForInterrupt sleeps until the interrupt channel is triggered 222 // or given timeout expires 223 func (le *leaderElectionSvcImpl) waitForInterrupt(timeout time.Duration) { 224 le.logger.Debug(le.id, ": Entering") 225 defer le.logger.Debug(le.id, ": Exiting") 226 le.Lock() 227 le.sleeping = true 228 le.Unlock() 229 230 select { 231 case <-le.interruptChan: 232 case <-le.stopChan: 233 le.stopChan <- struct{}{} 234 case <-time.After(timeout): 235 } 236 237 le.Lock() 238 le.sleeping = false 239 // We drain the interrupt channel 240 // because we might get 2 leadership declarations messages 241 // while sleeping, but we would only read 1 of them in the select block above 242 le.drainInterruptChannel() 243 le.Unlock() 244 } 245 246 func (le *leaderElectionSvcImpl) run() { 247 defer le.stopWG.Done() 248 for !le.shouldStop() { 249 if !le.isLeaderExists() { 250 le.leaderElection() 251 } 252 if le.shouldStop() { 253 return 254 } 255 if le.IsLeader() { 256 le.leader() 257 } else { 258 le.follower() 259 } 260 } 261 } 262 263 func (le *leaderElectionSvcImpl) leaderElection() { 264 le.logger.Debug(le.id, ": Entering") 265 defer le.logger.Debug(le.id, ": Exiting") 266 le.propose() 267 le.waitForInterrupt(getLeaderElectionDuration()) 268 // If someone declared itself as a leader, give up 269 // on trying to become a leader too 270 if le.isLeaderExists() { 271 le.logger.Debug(le.id, ": Some peer is already a leader") 272 return 273 } 274 // Leader doesn't exist, let's see if there is a better candidate than us 275 // for being a leader 276 for _, o := range le.proposals.ToArray() { 277 id := o.(string) 278 if bytes.Compare(peerID(id), le.id) < 0 { 279 return 280 } 281 } 282 // If we got here, there is no one that proposed being a leader 283 // that's a better candidate than us. 284 le.beLeader() 285 atomic.StoreInt32(&le.leaderExists, int32(1)) 286 } 287 288 // propose sends a leadership proposal message to remote peers 289 func (le *leaderElectionSvcImpl) propose() { 290 le.logger.Debug(le.id, ": Entering") 291 le.logger.Debug(le.id, ": Exiting") 292 leadershipProposal := le.adapter.CreateMessage(false) 293 le.adapter.Gossip(leadershipProposal) 294 } 295 296 func (le *leaderElectionSvcImpl) follower() { 297 le.logger.Debug(le.id, ": Entering") 298 defer le.logger.Debug(le.id, ": Exiting") 299 300 le.proposals.Clear() 301 atomic.StoreInt32(&le.leaderExists, int32(0)) 302 select { 303 case <-time.After(getLeaderAliveThreshold()): 304 case <-le.stopChan: 305 le.stopChan <- struct{}{} 306 } 307 } 308 309 func (le *leaderElectionSvcImpl) leader() { 310 leaderDeclaration := le.adapter.CreateMessage(true) 311 le.adapter.Gossip(leaderDeclaration) 312 le.waitForInterrupt(getLeadershipDeclarationInterval()) 313 } 314 315 // waitForMembershipStabilization waits for membership view to stabilize 316 // or until a time limit expires, or until a peer declares itself as a leader 317 func (le *leaderElectionSvcImpl) waitForMembershipStabilization(timeLimit time.Duration) { 318 le.logger.Debug(le.id, ": Entering") 319 defer le.logger.Debug(le.id, ": Exiting, peers found", len(le.adapter.Peers())) 320 endTime := time.Now().Add(timeLimit) 321 viewSize := len(le.adapter.Peers()) 322 for !le.shouldStop() { 323 time.Sleep(getMembershipSampleInterval()) 324 newSize := len(le.adapter.Peers()) 325 if newSize == viewSize || time.Now().After(endTime) || le.isLeaderExists() { 326 return 327 } 328 viewSize = newSize 329 } 330 } 331 332 // drainInterruptChannel clears the interruptChannel 333 // if needed 334 func (le *leaderElectionSvcImpl) drainInterruptChannel() { 335 if len(le.interruptChan) == 1 { 336 <-le.interruptChan 337 } 338 } 339 340 // isAlive returns whether peer of given id is considered alive 341 func (le *leaderElectionSvcImpl) isAlive(id peerID) bool { 342 for _, p := range le.adapter.Peers() { 343 if bytes.Equal(p.ID(), id) { 344 return true 345 } 346 } 347 return false 348 } 349 350 func (le *leaderElectionSvcImpl) isLeaderExists() bool { 351 return atomic.LoadInt32(&le.leaderExists) == int32(1) 352 } 353 354 // IsLeader returns whether this peer is a leader 355 func (le *leaderElectionSvcImpl) IsLeader() bool { 356 isLeader := atomic.LoadInt32(&le.isLeader) == int32(1) 357 le.logger.Debug(le.id, ": Returning", isLeader) 358 return isLeader 359 } 360 361 func (le *leaderElectionSvcImpl) beLeader() { 362 le.logger.Debug(le.id, ": Becoming a leader") 363 atomic.StoreInt32(&le.isLeader, int32(1)) 364 le.callback(true) 365 } 366 367 func (le *leaderElectionSvcImpl) stopBeingLeader() { 368 le.logger.Debug(le.id, "Stopped being a leader") 369 atomic.StoreInt32(&le.isLeader, int32(0)) 370 le.callback(false) 371 } 372 373 func (le *leaderElectionSvcImpl) shouldStop() bool { 374 return atomic.LoadInt32(&le.toDie) == int32(1) 375 } 376 377 // Stop stops the LeaderElectionService 378 func (le *leaderElectionSvcImpl) Stop() { 379 le.logger.Debug(le.id, ": Entering") 380 defer le.logger.Debug(le.id, ": Exiting") 381 atomic.StoreInt32(&le.toDie, int32(1)) 382 le.stopChan <- struct{}{} 383 le.stopWG.Wait() 384 } 385 386 // SetStartupGracePeriod configures startup grace period interval, 387 // the period of time to wait until election algorithm will start 388 func SetStartupGracePeriod(t time.Duration) { 389 viper.Set("peer.gossip.election.startupGracePeriod", t) 390 } 391 392 // SetMembershipSampleInterval setups/initializes the frequency the 393 // membership view should be checked 394 func SetMembershipSampleInterval(t time.Duration) { 395 viper.Set("peer.gossip.election.membershipSampleInterval", t) 396 } 397 398 // SetLeaderAliveThreshold configures leader election alive threshold 399 func SetLeaderAliveThreshold(t time.Duration) { 400 viper.Set("peer.gossip.election.leaderAliveThreshold", t) 401 } 402 403 // SetLeaderElectionDuration configures expected leadership election duration, 404 // interval to wait until leader election will be completed 405 func SetLeaderElectionDuration(t time.Duration) { 406 viper.Set("peer.gossip.election.leaderElectionDuration", t) 407 } 408 409 func getStartupGracePeriod() time.Duration { 410 return util.GetDurationOrDefault("peer.gossip.election.startupGracePeriod", time.Second*15) 411 } 412 413 func getMembershipSampleInterval() time.Duration { 414 return util.GetDurationOrDefault("peer.gossip.election.membershipSampleInterval", time.Second) 415 } 416 417 func getLeaderAliveThreshold() time.Duration { 418 return util.GetDurationOrDefault("peer.gossip.election.leaderAliveThreshold", time.Second*10) 419 } 420 421 func getLeadershipDeclarationInterval() time.Duration { 422 return time.Duration(getLeaderAliveThreshold() / 2) 423 } 424 425 func getLeaderElectionDuration() time.Duration { 426 return util.GetDurationOrDefault("peer.gossip.election.leaderElectionDuration", time.Second*5) 427 } 428 429 // GetMsgExpirationTimeout return leadership message expiration timeout 430 func GetMsgExpirationTimeout() time.Duration { 431 return getLeaderAliveThreshold() * 10 432 }