go.etcd.io/etcd@v3.3.27+incompatible/raft/raft.go (about) 1 // Copyright 2015 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package raft 16 17 import ( 18 "bytes" 19 "errors" 20 "fmt" 21 "math" 22 "math/rand" 23 "sort" 24 "strings" 25 "sync" 26 "time" 27 28 pb "github.com/coreos/etcd/raft/raftpb" 29 ) 30 31 // None is a placeholder node ID used when there is no leader. 32 const None uint64 = 0 33 const noLimit = math.MaxUint64 34 35 // Possible values for StateType. 36 const ( 37 StateFollower StateType = iota 38 StateCandidate 39 StateLeader 40 StatePreCandidate 41 numStates 42 ) 43 44 type ReadOnlyOption int 45 46 const ( 47 // ReadOnlySafe guarantees the linearizability of the read only request by 48 // communicating with the quorum. It is the default and suggested option. 49 ReadOnlySafe ReadOnlyOption = iota 50 // ReadOnlyLeaseBased ensures linearizability of the read only request by 51 // relying on the leader lease. It can be affected by clock drift. 52 // If the clock drift is unbounded, leader might keep the lease longer than it 53 // should (clock can move backward/pause without any bound). ReadIndex is not safe 54 // in that case. 55 ReadOnlyLeaseBased 56 ) 57 58 // Possible values for CampaignType 59 const ( 60 // campaignPreElection represents the first phase of a normal election when 61 // Config.PreVote is true. 62 campaignPreElection CampaignType = "CampaignPreElection" 63 // campaignElection represents a normal (time-based) election (the second phase 64 // of the election when Config.PreVote is true). 65 campaignElection CampaignType = "CampaignElection" 66 // campaignTransfer represents the type of leader transfer 67 campaignTransfer CampaignType = "CampaignTransfer" 68 ) 69 70 // lockedRand is a small wrapper around rand.Rand to provide 71 // synchronization. Only the methods needed by the code are exposed 72 // (e.g. Intn). 73 type lockedRand struct { 74 mu sync.Mutex 75 rand *rand.Rand 76 } 77 78 func (r *lockedRand) Intn(n int) int { 79 r.mu.Lock() 80 v := r.rand.Intn(n) 81 r.mu.Unlock() 82 return v 83 } 84 85 var globalRand = &lockedRand{ 86 rand: rand.New(rand.NewSource(time.Now().UnixNano())), 87 } 88 89 // CampaignType represents the type of campaigning 90 // the reason we use the type of string instead of uint64 91 // is because it's simpler to compare and fill in raft entries 92 type CampaignType string 93 94 // StateType represents the role of a node in a cluster. 95 type StateType uint64 96 97 var stmap = [...]string{ 98 "StateFollower", 99 "StateCandidate", 100 "StateLeader", 101 "StatePreCandidate", 102 } 103 104 func (st StateType) String() string { 105 return stmap[uint64(st)] 106 } 107 108 // Config contains the parameters to start a raft. 109 type Config struct { 110 // ID is the identity of the local raft. ID cannot be 0. 111 ID uint64 112 113 // peers contains the IDs of all nodes (including self) in the raft cluster. It 114 // should only be set when starting a new raft cluster. Restarting raft from 115 // previous configuration will panic if peers is set. peer is private and only 116 // used for testing right now. 117 peers []uint64 118 119 // learners contains the IDs of all leaner nodes (including self if the local node is a leaner) in the raft cluster. 120 // learners only receives entries from the leader node. It does not vote or promote itself. 121 learners []uint64 122 123 // ElectionTick is the number of Node.Tick invocations that must pass between 124 // elections. That is, if a follower does not receive any message from the 125 // leader of current term before ElectionTick has elapsed, it will become 126 // candidate and start an election. ElectionTick must be greater than 127 // HeartbeatTick. We suggest ElectionTick = 10 * HeartbeatTick to avoid 128 // unnecessary leader switching. 129 ElectionTick int 130 // HeartbeatTick is the number of Node.Tick invocations that must pass between 131 // heartbeats. That is, a leader sends heartbeat messages to maintain its 132 // leadership every HeartbeatTick ticks. 133 HeartbeatTick int 134 135 // Storage is the storage for raft. raft generates entries and states to be 136 // stored in storage. raft reads the persisted entries and states out of 137 // Storage when it needs. raft reads out the previous state and configuration 138 // out of storage when restarting. 139 Storage Storage 140 // Applied is the last applied index. It should only be set when restarting 141 // raft. raft will not return entries to the application smaller or equal to 142 // Applied. If Applied is unset when restarting, raft might return previous 143 // applied entries. This is a very application dependent configuration. 144 Applied uint64 145 146 // MaxSizePerMsg limits the max size of each append message. Smaller value 147 // lowers the raft recovery cost(initial probing and message lost during normal 148 // operation). On the other side, it might affect the throughput during normal 149 // replication. Note: math.MaxUint64 for unlimited, 0 for at most one entry per 150 // message. 151 MaxSizePerMsg uint64 152 // MaxInflightMsgs limits the max number of in-flight append messages during 153 // optimistic replication phase. The application transportation layer usually 154 // has its own sending buffer over TCP/UDP. Setting MaxInflightMsgs to avoid 155 // overflowing that sending buffer. TODO (xiangli): feedback to application to 156 // limit the proposal rate? 157 MaxInflightMsgs int 158 159 // CheckQuorum specifies if the leader should check quorum activity. Leader 160 // steps down when quorum is not active for an electionTimeout. 161 CheckQuorum bool 162 163 // PreVote enables the Pre-Vote algorithm described in raft thesis section 164 // 9.6. This prevents disruption when a node that has been partitioned away 165 // rejoins the cluster. 166 PreVote bool 167 168 // ReadOnlyOption specifies how the read only request is processed. 169 // 170 // ReadOnlySafe guarantees the linearizability of the read only request by 171 // communicating with the quorum. It is the default and suggested option. 172 // 173 // ReadOnlyLeaseBased ensures linearizability of the read only request by 174 // relying on the leader lease. It can be affected by clock drift. 175 // If the clock drift is unbounded, leader might keep the lease longer than it 176 // should (clock can move backward/pause without any bound). ReadIndex is not safe 177 // in that case. 178 // CheckQuorum MUST be enabled if ReadOnlyOption is ReadOnlyLeaseBased. 179 ReadOnlyOption ReadOnlyOption 180 181 // Logger is the logger used for raft log. For multinode which can host 182 // multiple raft group, each raft group can have its own logger 183 Logger Logger 184 185 // DisableProposalForwarding set to true means that followers will drop 186 // proposals, rather than forwarding them to the leader. One use case for 187 // this feature would be in a situation where the Raft leader is used to 188 // compute the data of a proposal, for example, adding a timestamp from a 189 // hybrid logical clock to data in a monotonically increasing way. Forwarding 190 // should be disabled to prevent a follower with an innaccurate hybrid 191 // logical clock from assigning the timestamp and then forwarding the data 192 // to the leader. 193 DisableProposalForwarding bool 194 } 195 196 func (c *Config) validate() error { 197 if c.ID == None { 198 return errors.New("cannot use none as id") 199 } 200 201 if c.HeartbeatTick <= 0 { 202 return errors.New("heartbeat tick must be greater than 0") 203 } 204 205 if c.ElectionTick <= c.HeartbeatTick { 206 return errors.New("election tick must be greater than heartbeat tick") 207 } 208 209 if c.Storage == nil { 210 return errors.New("storage cannot be nil") 211 } 212 213 if c.MaxInflightMsgs <= 0 { 214 return errors.New("max inflight messages must be greater than 0") 215 } 216 217 if c.Logger == nil { 218 c.Logger = raftLogger 219 } 220 221 if c.ReadOnlyOption == ReadOnlyLeaseBased && !c.CheckQuorum { 222 return errors.New("CheckQuorum must be enabled when ReadOnlyOption is ReadOnlyLeaseBased") 223 } 224 225 return nil 226 } 227 228 type raft struct { 229 id uint64 230 231 Term uint64 232 Vote uint64 233 234 readStates []ReadState 235 236 // the log 237 raftLog *raftLog 238 239 maxInflight int 240 maxMsgSize uint64 241 prs map[uint64]*Progress 242 learnerPrs map[uint64]*Progress 243 244 state StateType 245 246 // isLearner is true if the local raft node is a learner. 247 isLearner bool 248 249 votes map[uint64]bool 250 251 msgs []pb.Message 252 253 // the leader id 254 lead uint64 255 // leadTransferee is id of the leader transfer target when its value is not zero. 256 // Follow the procedure defined in raft thesis 3.10. 257 leadTransferee uint64 258 // New configuration is ignored if there exists unapplied configuration. 259 pendingConf bool 260 261 readOnly *readOnly 262 263 // number of ticks since it reached last electionTimeout when it is leader 264 // or candidate. 265 // number of ticks since it reached last electionTimeout or received a 266 // valid message from current leader when it is a follower. 267 electionElapsed int 268 269 // number of ticks since it reached last heartbeatTimeout. 270 // only leader keeps heartbeatElapsed. 271 heartbeatElapsed int 272 273 checkQuorum bool 274 preVote bool 275 276 heartbeatTimeout int 277 electionTimeout int 278 // randomizedElectionTimeout is a random number between 279 // [electiontimeout, 2 * electiontimeout - 1]. It gets reset 280 // when raft changes its state to follower or candidate. 281 randomizedElectionTimeout int 282 disableProposalForwarding bool 283 284 tick func() 285 step stepFunc 286 287 logger Logger 288 } 289 290 func newRaft(c *Config) *raft { 291 if err := c.validate(); err != nil { 292 panic(err.Error()) 293 } 294 raftlog := newLog(c.Storage, c.Logger) 295 hs, cs, err := c.Storage.InitialState() 296 if err != nil { 297 panic(err) // TODO(bdarnell) 298 } 299 peers := c.peers 300 learners := c.learners 301 if len(cs.Nodes) > 0 || len(cs.Learners) > 0 { 302 if len(peers) > 0 || len(learners) > 0 { 303 // TODO(bdarnell): the peers argument is always nil except in 304 // tests; the argument should be removed and these tests should be 305 // updated to specify their nodes through a snapshot. 306 panic("cannot specify both newRaft(peers, learners) and ConfState.(Nodes, Learners)") 307 } 308 peers = cs.Nodes 309 learners = cs.Learners 310 } 311 r := &raft{ 312 id: c.ID, 313 lead: None, 314 isLearner: false, 315 raftLog: raftlog, 316 maxMsgSize: c.MaxSizePerMsg, 317 maxInflight: c.MaxInflightMsgs, 318 prs: make(map[uint64]*Progress), 319 learnerPrs: make(map[uint64]*Progress), 320 electionTimeout: c.ElectionTick, 321 heartbeatTimeout: c.HeartbeatTick, 322 logger: c.Logger, 323 checkQuorum: c.CheckQuorum, 324 preVote: c.PreVote, 325 readOnly: newReadOnly(c.ReadOnlyOption), 326 disableProposalForwarding: c.DisableProposalForwarding, 327 } 328 for _, p := range peers { 329 r.prs[p] = &Progress{Next: 1, ins: newInflights(r.maxInflight)} 330 } 331 for _, p := range learners { 332 if _, ok := r.prs[p]; ok { 333 panic(fmt.Sprintf("node %x is in both learner and peer list", p)) 334 } 335 r.learnerPrs[p] = &Progress{Next: 1, ins: newInflights(r.maxInflight), IsLearner: true} 336 if r.id == p { 337 r.isLearner = true 338 } 339 } 340 341 if !isHardStateEqual(hs, emptyState) { 342 r.loadState(hs) 343 } 344 if c.Applied > 0 { 345 raftlog.appliedTo(c.Applied) 346 } 347 r.becomeFollower(r.Term, None) 348 349 var nodesStrs []string 350 for _, n := range r.nodes() { 351 nodesStrs = append(nodesStrs, fmt.Sprintf("%x", n)) 352 } 353 354 r.logger.Infof("newRaft %x [peers: [%s], term: %d, commit: %d, applied: %d, lastindex: %d, lastterm: %d]", 355 r.id, strings.Join(nodesStrs, ","), r.Term, r.raftLog.committed, r.raftLog.applied, r.raftLog.lastIndex(), r.raftLog.lastTerm()) 356 return r 357 } 358 359 func (r *raft) hasLeader() bool { return r.lead != None } 360 361 func (r *raft) softState() *SoftState { return &SoftState{Lead: r.lead, RaftState: r.state} } 362 363 func (r *raft) hardState() pb.HardState { 364 return pb.HardState{ 365 Term: r.Term, 366 Vote: r.Vote, 367 Commit: r.raftLog.committed, 368 } 369 } 370 371 func (r *raft) quorum() int { return len(r.prs)/2 + 1 } 372 373 func (r *raft) nodes() []uint64 { 374 nodes := make([]uint64, 0, len(r.prs)+len(r.learnerPrs)) 375 for id := range r.prs { 376 nodes = append(nodes, id) 377 } 378 for id := range r.learnerPrs { 379 nodes = append(nodes, id) 380 } 381 sort.Sort(uint64Slice(nodes)) 382 return nodes 383 } 384 385 // send persists state to stable storage and then sends to its mailbox. 386 func (r *raft) send(m pb.Message) { 387 m.From = r.id 388 if m.Type == pb.MsgVote || m.Type == pb.MsgVoteResp || m.Type == pb.MsgPreVote || m.Type == pb.MsgPreVoteResp { 389 if m.Term == 0 { 390 // All {pre-,}campaign messages need to have the term set when 391 // sending. 392 // - MsgVote: m.Term is the term the node is campaigning for, 393 // non-zero as we increment the term when campaigning. 394 // - MsgVoteResp: m.Term is the new r.Term if the MsgVote was 395 // granted, non-zero for the same reason MsgVote is 396 // - MsgPreVote: m.Term is the term the node will campaign, 397 // non-zero as we use m.Term to indicate the next term we'll be 398 // campaigning for 399 // - MsgPreVoteResp: m.Term is the term received in the original 400 // MsgPreVote if the pre-vote was granted, non-zero for the 401 // same reasons MsgPreVote is 402 panic(fmt.Sprintf("term should be set when sending %s", m.Type)) 403 } 404 } else { 405 if m.Term != 0 { 406 panic(fmt.Sprintf("term should not be set when sending %s (was %d)", m.Type, m.Term)) 407 } 408 // do not attach term to MsgProp, MsgReadIndex 409 // proposals are a way to forward to the leader and 410 // should be treated as local message. 411 // MsgReadIndex is also forwarded to leader. 412 if m.Type != pb.MsgProp && m.Type != pb.MsgReadIndex { 413 m.Term = r.Term 414 } 415 } 416 r.msgs = append(r.msgs, m) 417 } 418 419 func (r *raft) getProgress(id uint64) *Progress { 420 if pr, ok := r.prs[id]; ok { 421 return pr 422 } 423 424 return r.learnerPrs[id] 425 } 426 427 // sendAppend sends RPC, with entries to the given peer. 428 func (r *raft) sendAppend(to uint64) { 429 pr := r.getProgress(to) 430 if pr.IsPaused() { 431 return 432 } 433 m := pb.Message{} 434 m.To = to 435 436 term, errt := r.raftLog.term(pr.Next - 1) 437 ents, erre := r.raftLog.entries(pr.Next, r.maxMsgSize) 438 439 if errt != nil || erre != nil { // send snapshot if we failed to get term or entries 440 if !pr.RecentActive { 441 r.logger.Debugf("ignore sending snapshot to %x since it is not recently active", to) 442 return 443 } 444 445 m.Type = pb.MsgSnap 446 snapshot, err := r.raftLog.snapshot() 447 if err != nil { 448 if err == ErrSnapshotTemporarilyUnavailable { 449 r.logger.Debugf("%x failed to send snapshot to %x because snapshot is temporarily unavailable", r.id, to) 450 return 451 } 452 panic(err) // TODO(bdarnell) 453 } 454 if IsEmptySnap(snapshot) { 455 panic("need non-empty snapshot") 456 } 457 m.Snapshot = snapshot 458 sindex, sterm := snapshot.Metadata.Index, snapshot.Metadata.Term 459 r.logger.Debugf("%x [firstindex: %d, commit: %d] sent snapshot[index: %d, term: %d] to %x [%s]", 460 r.id, r.raftLog.firstIndex(), r.raftLog.committed, sindex, sterm, to, pr) 461 pr.becomeSnapshot(sindex) 462 r.logger.Debugf("%x paused sending replication messages to %x [%s]", r.id, to, pr) 463 } else { 464 m.Type = pb.MsgApp 465 m.Index = pr.Next - 1 466 m.LogTerm = term 467 m.Entries = ents 468 m.Commit = r.raftLog.committed 469 if n := len(m.Entries); n != 0 { 470 switch pr.State { 471 // optimistically increase the next when in ProgressStateReplicate 472 case ProgressStateReplicate: 473 last := m.Entries[n-1].Index 474 pr.optimisticUpdate(last) 475 pr.ins.add(last) 476 case ProgressStateProbe: 477 pr.pause() 478 default: 479 r.logger.Panicf("%x is sending append in unhandled state %s", r.id, pr.State) 480 } 481 } 482 } 483 r.send(m) 484 } 485 486 // sendHeartbeat sends an empty MsgApp 487 func (r *raft) sendHeartbeat(to uint64, ctx []byte) { 488 // Attach the commit as min(to.matched, r.committed). 489 // When the leader sends out heartbeat message, 490 // the receiver(follower) might not be matched with the leader 491 // or it might not have all the committed entries. 492 // The leader MUST NOT forward the follower's commit to 493 // an unmatched index. 494 commit := min(r.getProgress(to).Match, r.raftLog.committed) 495 m := pb.Message{ 496 To: to, 497 Type: pb.MsgHeartbeat, 498 Commit: commit, 499 Context: ctx, 500 } 501 502 r.send(m) 503 } 504 505 func (r *raft) forEachProgress(f func(id uint64, pr *Progress)) { 506 for id, pr := range r.prs { 507 f(id, pr) 508 } 509 510 for id, pr := range r.learnerPrs { 511 f(id, pr) 512 } 513 } 514 515 // bcastAppend sends RPC, with entries to all peers that are not up-to-date 516 // according to the progress recorded in r.prs. 517 func (r *raft) bcastAppend() { 518 r.forEachProgress(func(id uint64, _ *Progress) { 519 if id == r.id { 520 return 521 } 522 523 r.sendAppend(id) 524 }) 525 } 526 527 // bcastHeartbeat sends RPC, without entries to all the peers. 528 func (r *raft) bcastHeartbeat() { 529 lastCtx := r.readOnly.lastPendingRequestCtx() 530 if len(lastCtx) == 0 { 531 r.bcastHeartbeatWithCtx(nil) 532 } else { 533 r.bcastHeartbeatWithCtx([]byte(lastCtx)) 534 } 535 } 536 537 func (r *raft) bcastHeartbeatWithCtx(ctx []byte) { 538 r.forEachProgress(func(id uint64, _ *Progress) { 539 if id == r.id { 540 return 541 } 542 r.sendHeartbeat(id, ctx) 543 }) 544 } 545 546 // maybeCommit attempts to advance the commit index. Returns true if 547 // the commit index changed (in which case the caller should call 548 // r.bcastAppend). 549 func (r *raft) maybeCommit() bool { 550 // TODO(bmizerany): optimize.. Currently naive 551 mis := make(uint64Slice, 0, len(r.prs)) 552 for _, p := range r.prs { 553 mis = append(mis, p.Match) 554 } 555 sort.Sort(sort.Reverse(mis)) 556 mci := mis[r.quorum()-1] 557 return r.raftLog.maybeCommit(mci, r.Term) 558 } 559 560 func (r *raft) reset(term uint64) { 561 if r.Term != term { 562 r.Term = term 563 r.Vote = None 564 } 565 r.lead = None 566 567 r.electionElapsed = 0 568 r.heartbeatElapsed = 0 569 r.resetRandomizedElectionTimeout() 570 571 r.abortLeaderTransfer() 572 573 r.votes = make(map[uint64]bool) 574 r.forEachProgress(func(id uint64, pr *Progress) { 575 *pr = Progress{Next: r.raftLog.lastIndex() + 1, ins: newInflights(r.maxInflight), IsLearner: pr.IsLearner} 576 if id == r.id { 577 pr.Match = r.raftLog.lastIndex() 578 } 579 }) 580 581 r.pendingConf = false 582 r.readOnly = newReadOnly(r.readOnly.option) 583 } 584 585 func (r *raft) appendEntry(es ...pb.Entry) { 586 li := r.raftLog.lastIndex() 587 for i := range es { 588 es[i].Term = r.Term 589 es[i].Index = li + 1 + uint64(i) 590 } 591 r.raftLog.append(es...) 592 r.getProgress(r.id).maybeUpdate(r.raftLog.lastIndex()) 593 // Regardless of maybeCommit's return, our caller will call bcastAppend. 594 r.maybeCommit() 595 } 596 597 // tickElection is run by followers and candidates after r.electionTimeout. 598 func (r *raft) tickElection() { 599 r.electionElapsed++ 600 601 if r.promotable() && r.pastElectionTimeout() { 602 r.electionElapsed = 0 603 r.Step(pb.Message{From: r.id, Type: pb.MsgHup}) 604 } 605 } 606 607 // tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout. 608 func (r *raft) tickHeartbeat() { 609 r.heartbeatElapsed++ 610 r.electionElapsed++ 611 612 if r.electionElapsed >= r.electionTimeout { 613 r.electionElapsed = 0 614 if r.checkQuorum { 615 r.Step(pb.Message{From: r.id, Type: pb.MsgCheckQuorum}) 616 } 617 // If current leader cannot transfer leadership in electionTimeout, it becomes leader again. 618 if r.state == StateLeader && r.leadTransferee != None { 619 r.abortLeaderTransfer() 620 } 621 } 622 623 if r.state != StateLeader { 624 return 625 } 626 627 if r.heartbeatElapsed >= r.heartbeatTimeout { 628 r.heartbeatElapsed = 0 629 r.Step(pb.Message{From: r.id, Type: pb.MsgBeat}) 630 } 631 } 632 633 func (r *raft) becomeFollower(term uint64, lead uint64) { 634 r.step = stepFollower 635 r.reset(term) 636 r.tick = r.tickElection 637 r.lead = lead 638 r.state = StateFollower 639 r.logger.Infof("%x became follower at term %d", r.id, r.Term) 640 } 641 642 func (r *raft) becomeCandidate() { 643 // TODO(xiangli) remove the panic when the raft implementation is stable 644 if r.state == StateLeader { 645 panic("invalid transition [leader -> candidate]") 646 } 647 r.step = stepCandidate 648 r.reset(r.Term + 1) 649 r.tick = r.tickElection 650 r.Vote = r.id 651 r.state = StateCandidate 652 r.logger.Infof("%x became candidate at term %d", r.id, r.Term) 653 } 654 655 func (r *raft) becomePreCandidate() { 656 // TODO(xiangli) remove the panic when the raft implementation is stable 657 if r.state == StateLeader { 658 panic("invalid transition [leader -> pre-candidate]") 659 } 660 // Becoming a pre-candidate changes our step functions and state, 661 // but doesn't change anything else. In particular it does not increase 662 // r.Term or change r.Vote. 663 r.step = stepCandidate 664 r.votes = make(map[uint64]bool) 665 r.tick = r.tickElection 666 r.lead = None 667 r.state = StatePreCandidate 668 r.logger.Infof("%x became pre-candidate at term %d", r.id, r.Term) 669 } 670 671 func (r *raft) becomeLeader() { 672 // TODO(xiangli) remove the panic when the raft implementation is stable 673 if r.state == StateFollower { 674 panic("invalid transition [follower -> leader]") 675 } 676 r.step = stepLeader 677 r.reset(r.Term) 678 r.tick = r.tickHeartbeat 679 r.lead = r.id 680 r.state = StateLeader 681 ents, err := r.raftLog.entries(r.raftLog.committed+1, noLimit) 682 if err != nil { 683 r.logger.Panicf("unexpected error getting uncommitted entries (%v)", err) 684 } 685 686 nconf := numOfPendingConf(ents) 687 if nconf > 1 { 688 panic("unexpected multiple uncommitted config entry") 689 } 690 if nconf == 1 { 691 r.pendingConf = true 692 } 693 694 r.appendEntry(pb.Entry{Data: nil}) 695 r.logger.Infof("%x became leader at term %d", r.id, r.Term) 696 } 697 698 func (r *raft) campaign(t CampaignType) { 699 var term uint64 700 var voteMsg pb.MessageType 701 if t == campaignPreElection { 702 r.becomePreCandidate() 703 voteMsg = pb.MsgPreVote 704 // PreVote RPCs are sent for the next term before we've incremented r.Term. 705 term = r.Term + 1 706 } else { 707 r.becomeCandidate() 708 voteMsg = pb.MsgVote 709 term = r.Term 710 } 711 if r.quorum() == r.poll(r.id, voteRespMsgType(voteMsg), true) { 712 // We won the election after voting for ourselves (which must mean that 713 // this is a single-node cluster). Advance to the next state. 714 if t == campaignPreElection { 715 r.campaign(campaignElection) 716 } else { 717 r.becomeLeader() 718 } 719 return 720 } 721 for id := range r.prs { 722 if id == r.id { 723 continue 724 } 725 r.logger.Infof("%x [logterm: %d, index: %d] sent %s request to %x at term %d", 726 r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), voteMsg, id, r.Term) 727 728 var ctx []byte 729 if t == campaignTransfer { 730 ctx = []byte(t) 731 } 732 r.send(pb.Message{Term: term, To: id, Type: voteMsg, Index: r.raftLog.lastIndex(), LogTerm: r.raftLog.lastTerm(), Context: ctx}) 733 } 734 } 735 736 func (r *raft) poll(id uint64, t pb.MessageType, v bool) (granted int) { 737 if v { 738 r.logger.Infof("%x received %s from %x at term %d", r.id, t, id, r.Term) 739 } else { 740 r.logger.Infof("%x received %s rejection from %x at term %d", r.id, t, id, r.Term) 741 } 742 if _, ok := r.votes[id]; !ok { 743 r.votes[id] = v 744 } 745 for _, vv := range r.votes { 746 if vv { 747 granted++ 748 } 749 } 750 return granted 751 } 752 753 func (r *raft) Step(m pb.Message) error { 754 // Handle the message term, which may result in our stepping down to a follower. 755 switch { 756 case m.Term == 0: 757 // local message 758 case m.Term > r.Term: 759 if m.Type == pb.MsgVote || m.Type == pb.MsgPreVote { 760 force := bytes.Equal(m.Context, []byte(campaignTransfer)) 761 inLease := r.checkQuorum && r.lead != None && r.electionElapsed < r.electionTimeout 762 if !force && inLease { 763 // If a server receives a RequestVote request within the minimum election timeout 764 // of hearing from a current leader, it does not update its term or grant its vote 765 r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] ignored %s from %x [logterm: %d, index: %d] at term %d: lease is not expired (remaining ticks: %d)", 766 r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term, r.electionTimeout-r.electionElapsed) 767 return nil 768 } 769 } 770 switch { 771 case m.Type == pb.MsgPreVote: 772 // Never change our term in response to a PreVote 773 case m.Type == pb.MsgPreVoteResp && !m.Reject: 774 // We send pre-vote requests with a term in our future. If the 775 // pre-vote is granted, we will increment our term when we get a 776 // quorum. If it is not, the term comes from the node that 777 // rejected our vote so we should become a follower at the new 778 // term. 779 default: 780 r.logger.Infof("%x [term: %d] received a %s message with higher term from %x [term: %d]", 781 r.id, r.Term, m.Type, m.From, m.Term) 782 if m.Type == pb.MsgApp || m.Type == pb.MsgHeartbeat || m.Type == pb.MsgSnap { 783 r.becomeFollower(m.Term, m.From) 784 } else { 785 r.becomeFollower(m.Term, None) 786 } 787 } 788 789 case m.Term < r.Term: 790 if r.checkQuorum && (m.Type == pb.MsgHeartbeat || m.Type == pb.MsgApp) { 791 // We have received messages from a leader at a lower term. It is possible 792 // that these messages were simply delayed in the network, but this could 793 // also mean that this node has advanced its term number during a network 794 // partition, and it is now unable to either win an election or to rejoin 795 // the majority on the old term. If checkQuorum is false, this will be 796 // handled by incrementing term numbers in response to MsgVote with a 797 // higher term, but if checkQuorum is true we may not advance the term on 798 // MsgVote and must generate other messages to advance the term. The net 799 // result of these two features is to minimize the disruption caused by 800 // nodes that have been removed from the cluster's configuration: a 801 // removed node will send MsgVotes (or MsgPreVotes) which will be ignored, 802 // but it will not receive MsgApp or MsgHeartbeat, so it will not create 803 // disruptive term increases 804 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp}) 805 } else { 806 // ignore other cases 807 r.logger.Infof("%x [term: %d] ignored a %s message with lower term from %x [term: %d]", 808 r.id, r.Term, m.Type, m.From, m.Term) 809 } 810 return nil 811 } 812 813 switch m.Type { 814 case pb.MsgHup: 815 if r.state != StateLeader { 816 ents, err := r.raftLog.slice(r.raftLog.applied+1, r.raftLog.committed+1, noLimit) 817 if err != nil { 818 r.logger.Panicf("unexpected error getting unapplied entries (%v)", err) 819 } 820 if n := numOfPendingConf(ents); n != 0 && r.raftLog.committed > r.raftLog.applied { 821 r.logger.Warningf("%x cannot campaign at term %d since there are still %d pending configuration changes to apply", r.id, r.Term, n) 822 return nil 823 } 824 825 r.logger.Infof("%x is starting a new election at term %d", r.id, r.Term) 826 if r.preVote { 827 r.campaign(campaignPreElection) 828 } else { 829 r.campaign(campaignElection) 830 } 831 } else { 832 r.logger.Debugf("%x ignoring MsgHup because already leader", r.id) 833 } 834 835 case pb.MsgVote, pb.MsgPreVote: 836 if r.isLearner { 837 // TODO: learner may need to vote, in case of node down when confchange. 838 r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] ignored %s from %x [logterm: %d, index: %d] at term %d: learner can not vote", 839 r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term) 840 return nil 841 } 842 // The m.Term > r.Term clause is for MsgPreVote. For MsgVote m.Term should 843 // always equal r.Term. 844 if (r.Vote == None || m.Term > r.Term || r.Vote == m.From) && r.raftLog.isUpToDate(m.Index, m.LogTerm) { 845 r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] cast %s for %x [logterm: %d, index: %d] at term %d", 846 r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term) 847 // When responding to Msg{Pre,}Vote messages we include the term 848 // from the message, not the local term. To see why consider the 849 // case where a single node was previously partitioned away and 850 // it's local term is now of date. If we include the local term 851 // (recall that for pre-votes we don't update the local term), the 852 // (pre-)campaigning node on the other end will proceed to ignore 853 // the message (it ignores all out of date messages). 854 // The term in the original message and current local term are the 855 // same in the case of regular votes, but different for pre-votes. 856 r.send(pb.Message{To: m.From, Term: m.Term, Type: voteRespMsgType(m.Type)}) 857 if m.Type == pb.MsgVote { 858 // Only record real votes. 859 r.electionElapsed = 0 860 r.Vote = m.From 861 } 862 } else { 863 r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] rejected %s from %x [logterm: %d, index: %d] at term %d", 864 r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term) 865 r.send(pb.Message{To: m.From, Term: r.Term, Type: voteRespMsgType(m.Type), Reject: true}) 866 } 867 868 default: 869 r.step(r, m) 870 } 871 return nil 872 } 873 874 type stepFunc func(r *raft, m pb.Message) 875 876 func stepLeader(r *raft, m pb.Message) { 877 // These message types do not require any progress for m.From. 878 switch m.Type { 879 case pb.MsgBeat: 880 r.bcastHeartbeat() 881 return 882 case pb.MsgCheckQuorum: 883 if !r.checkQuorumActive() { 884 r.logger.Warningf("%x stepped down to follower since quorum is not active", r.id) 885 r.becomeFollower(r.Term, None) 886 } 887 return 888 case pb.MsgProp: 889 if len(m.Entries) == 0 { 890 r.logger.Panicf("%x stepped empty MsgProp", r.id) 891 } 892 if _, ok := r.prs[r.id]; !ok { 893 // If we are not currently a member of the range (i.e. this node 894 // was removed from the configuration while serving as leader), 895 // drop any new proposals. 896 return 897 } 898 if r.leadTransferee != None { 899 r.logger.Debugf("%x [term %d] transfer leadership to %x is in progress; dropping proposal", r.id, r.Term, r.leadTransferee) 900 return 901 } 902 903 for i, e := range m.Entries { 904 if e.Type == pb.EntryConfChange { 905 if r.pendingConf { 906 r.logger.Infof("propose conf %s ignored since pending unapplied configuration", e.String()) 907 m.Entries[i] = pb.Entry{Type: pb.EntryNormal} 908 } 909 r.pendingConf = true 910 } 911 } 912 r.appendEntry(m.Entries...) 913 r.bcastAppend() 914 return 915 case pb.MsgReadIndex: 916 if r.quorum() > 1 { 917 if r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(r.raftLog.committed)) != r.Term { 918 // Reject read only request when this leader has not committed any log entry at its term. 919 return 920 } 921 922 // thinking: use an interally defined context instead of the user given context. 923 // We can express this in terms of the term and index instead of a user-supplied value. 924 // This would allow multiple reads to piggyback on the same message. 925 switch r.readOnly.option { 926 case ReadOnlySafe: 927 r.readOnly.addRequest(r.raftLog.committed, m) 928 r.bcastHeartbeatWithCtx(m.Entries[0].Data) 929 case ReadOnlyLeaseBased: 930 ri := r.raftLog.committed 931 if m.From == None || m.From == r.id { // from local member 932 r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data}) 933 } else { 934 r.send(pb.Message{To: m.From, Type: pb.MsgReadIndexResp, Index: ri, Entries: m.Entries}) 935 } 936 } 937 } else { 938 r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data}) 939 } 940 941 return 942 } 943 944 // All other message types require a progress for m.From (pr). 945 pr := r.getProgress(m.From) 946 if pr == nil { 947 r.logger.Debugf("%x no progress available for %x", r.id, m.From) 948 return 949 } 950 switch m.Type { 951 case pb.MsgAppResp: 952 pr.RecentActive = true 953 954 if m.Reject { 955 r.logger.Debugf("%x received msgApp rejection(lastindex: %d) from %x for index %d", 956 r.id, m.RejectHint, m.From, m.Index) 957 if pr.maybeDecrTo(m.Index, m.RejectHint) { 958 r.logger.Debugf("%x decreased progress of %x to [%s]", r.id, m.From, pr) 959 if pr.State == ProgressStateReplicate { 960 pr.becomeProbe() 961 } 962 r.sendAppend(m.From) 963 } 964 } else { 965 oldPaused := pr.IsPaused() 966 if pr.maybeUpdate(m.Index) { 967 switch { 968 case pr.State == ProgressStateProbe: 969 pr.becomeReplicate() 970 case pr.State == ProgressStateSnapshot && pr.needSnapshotAbort(): 971 r.logger.Debugf("%x snapshot aborted, resumed sending replication messages to %x [%s]", r.id, m.From, pr) 972 pr.becomeProbe() 973 case pr.State == ProgressStateReplicate: 974 pr.ins.freeTo(m.Index) 975 } 976 977 if r.maybeCommit() { 978 r.bcastAppend() 979 } else if oldPaused { 980 // update() reset the wait state on this node. If we had delayed sending 981 // an update before, send it now. 982 r.sendAppend(m.From) 983 } 984 // Transfer leadership is in progress. 985 if m.From == r.leadTransferee && pr.Match == r.raftLog.lastIndex() { 986 r.logger.Infof("%x sent MsgTimeoutNow to %x after received MsgAppResp", r.id, m.From) 987 r.sendTimeoutNow(m.From) 988 } 989 } 990 } 991 case pb.MsgHeartbeatResp: 992 pr.RecentActive = true 993 pr.resume() 994 995 // free one slot for the full inflights window to allow progress. 996 if pr.State == ProgressStateReplicate && pr.ins.full() { 997 pr.ins.freeFirstOne() 998 } 999 if pr.Match < r.raftLog.lastIndex() { 1000 r.sendAppend(m.From) 1001 } 1002 1003 if r.readOnly.option != ReadOnlySafe || len(m.Context) == 0 { 1004 return 1005 } 1006 1007 ackCount := r.readOnly.recvAck(m) 1008 if ackCount < r.quorum() { 1009 return 1010 } 1011 1012 rss := r.readOnly.advance(m) 1013 for _, rs := range rss { 1014 req := rs.req 1015 if req.From == None || req.From == r.id { // from local member 1016 r.readStates = append(r.readStates, ReadState{Index: rs.index, RequestCtx: req.Entries[0].Data}) 1017 } else { 1018 r.send(pb.Message{To: req.From, Type: pb.MsgReadIndexResp, Index: rs.index, Entries: req.Entries}) 1019 } 1020 } 1021 case pb.MsgSnapStatus: 1022 if pr.State != ProgressStateSnapshot { 1023 return 1024 } 1025 if !m.Reject { 1026 pr.becomeProbe() 1027 r.logger.Debugf("%x snapshot succeeded, resumed sending replication messages to %x [%s]", r.id, m.From, pr) 1028 } else { 1029 pr.snapshotFailure() 1030 pr.becomeProbe() 1031 r.logger.Debugf("%x snapshot failed, resumed sending replication messages to %x [%s]", r.id, m.From, pr) 1032 } 1033 // If snapshot finish, wait for the msgAppResp from the remote node before sending 1034 // out the next msgApp. 1035 // If snapshot failure, wait for a heartbeat interval before next try 1036 pr.pause() 1037 case pb.MsgUnreachable: 1038 // During optimistic replication, if the remote becomes unreachable, 1039 // there is huge probability that a MsgApp is lost. 1040 if pr.State == ProgressStateReplicate { 1041 pr.becomeProbe() 1042 } 1043 r.logger.Debugf("%x failed to send message to %x because it is unreachable [%s]", r.id, m.From, pr) 1044 case pb.MsgTransferLeader: 1045 if pr.IsLearner { 1046 r.logger.Debugf("%x is learner. Ignored transferring leadership", r.id) 1047 return 1048 } 1049 leadTransferee := m.From 1050 lastLeadTransferee := r.leadTransferee 1051 if lastLeadTransferee != None { 1052 if lastLeadTransferee == leadTransferee { 1053 r.logger.Infof("%x [term %d] transfer leadership to %x is in progress, ignores request to same node %x", 1054 r.id, r.Term, leadTransferee, leadTransferee) 1055 return 1056 } 1057 r.abortLeaderTransfer() 1058 r.logger.Infof("%x [term %d] abort previous transferring leadership to %x", r.id, r.Term, lastLeadTransferee) 1059 } 1060 if leadTransferee == r.id { 1061 r.logger.Debugf("%x is already leader. Ignored transferring leadership to self", r.id) 1062 return 1063 } 1064 // Transfer leadership to third party. 1065 r.logger.Infof("%x [term %d] starts to transfer leadership to %x", r.id, r.Term, leadTransferee) 1066 // Transfer leadership should be finished in one electionTimeout, so reset r.electionElapsed. 1067 r.electionElapsed = 0 1068 r.leadTransferee = leadTransferee 1069 if pr.Match == r.raftLog.lastIndex() { 1070 r.sendTimeoutNow(leadTransferee) 1071 r.logger.Infof("%x sends MsgTimeoutNow to %x immediately as %x already has up-to-date log", r.id, leadTransferee, leadTransferee) 1072 } else { 1073 r.sendAppend(leadTransferee) 1074 } 1075 } 1076 } 1077 1078 // stepCandidate is shared by StateCandidate and StatePreCandidate; the difference is 1079 // whether they respond to MsgVoteResp or MsgPreVoteResp. 1080 func stepCandidate(r *raft, m pb.Message) { 1081 // Only handle vote responses corresponding to our candidacy (while in 1082 // StateCandidate, we may get stale MsgPreVoteResp messages in this term from 1083 // our pre-candidate state). 1084 var myVoteRespType pb.MessageType 1085 if r.state == StatePreCandidate { 1086 myVoteRespType = pb.MsgPreVoteResp 1087 } else { 1088 myVoteRespType = pb.MsgVoteResp 1089 } 1090 switch m.Type { 1091 case pb.MsgProp: 1092 r.logger.Infof("%x no leader at term %d; dropping proposal", r.id, r.Term) 1093 return 1094 case pb.MsgApp: 1095 r.becomeFollower(r.Term, m.From) 1096 r.handleAppendEntries(m) 1097 case pb.MsgHeartbeat: 1098 r.becomeFollower(r.Term, m.From) 1099 r.handleHeartbeat(m) 1100 case pb.MsgSnap: 1101 r.becomeFollower(m.Term, m.From) 1102 r.handleSnapshot(m) 1103 case myVoteRespType: 1104 gr := r.poll(m.From, m.Type, !m.Reject) 1105 r.logger.Infof("%x [quorum:%d] has received %d %s votes and %d vote rejections", r.id, r.quorum(), gr, m.Type, len(r.votes)-gr) 1106 switch r.quorum() { 1107 case gr: 1108 if r.state == StatePreCandidate { 1109 r.campaign(campaignElection) 1110 } else { 1111 r.becomeLeader() 1112 r.bcastAppend() 1113 } 1114 case len(r.votes) - gr: 1115 r.becomeFollower(r.Term, None) 1116 } 1117 case pb.MsgTimeoutNow: 1118 r.logger.Debugf("%x [term %d state %v] ignored MsgTimeoutNow from %x", r.id, r.Term, r.state, m.From) 1119 } 1120 } 1121 1122 func stepFollower(r *raft, m pb.Message) { 1123 switch m.Type { 1124 case pb.MsgProp: 1125 if r.lead == None { 1126 r.logger.Infof("%x no leader at term %d; dropping proposal", r.id, r.Term) 1127 return 1128 } else if r.disableProposalForwarding { 1129 r.logger.Infof("%x not forwarding to leader %x at term %d; dropping proposal", r.id, r.lead, r.Term) 1130 return 1131 } 1132 m.To = r.lead 1133 r.send(m) 1134 case pb.MsgApp: 1135 r.electionElapsed = 0 1136 r.lead = m.From 1137 r.handleAppendEntries(m) 1138 case pb.MsgHeartbeat: 1139 r.electionElapsed = 0 1140 r.lead = m.From 1141 r.handleHeartbeat(m) 1142 case pb.MsgSnap: 1143 r.electionElapsed = 0 1144 r.lead = m.From 1145 r.handleSnapshot(m) 1146 case pb.MsgTransferLeader: 1147 if r.lead == None { 1148 r.logger.Infof("%x no leader at term %d; dropping leader transfer msg", r.id, r.Term) 1149 return 1150 } 1151 m.To = r.lead 1152 r.send(m) 1153 case pb.MsgTimeoutNow: 1154 if r.promotable() { 1155 r.logger.Infof("%x [term %d] received MsgTimeoutNow from %x and starts an election to get leadership.", r.id, r.Term, m.From) 1156 // Leadership transfers never use pre-vote even if r.preVote is true; we 1157 // know we are not recovering from a partition so there is no need for the 1158 // extra round trip. 1159 r.campaign(campaignTransfer) 1160 } else { 1161 r.logger.Infof("%x received MsgTimeoutNow from %x but is not promotable", r.id, m.From) 1162 } 1163 case pb.MsgReadIndex: 1164 if r.lead == None { 1165 r.logger.Infof("%x no leader at term %d; dropping index reading msg", r.id, r.Term) 1166 return 1167 } 1168 m.To = r.lead 1169 r.send(m) 1170 case pb.MsgReadIndexResp: 1171 if len(m.Entries) != 1 { 1172 r.logger.Errorf("%x invalid format of MsgReadIndexResp from %x, entries count: %d", r.id, m.From, len(m.Entries)) 1173 return 1174 } 1175 r.readStates = append(r.readStates, ReadState{Index: m.Index, RequestCtx: m.Entries[0].Data}) 1176 } 1177 } 1178 1179 func (r *raft) handleAppendEntries(m pb.Message) { 1180 if m.Index < r.raftLog.committed { 1181 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed}) 1182 return 1183 } 1184 1185 if mlastIndex, ok := r.raftLog.maybeAppend(m.Index, m.LogTerm, m.Commit, m.Entries...); ok { 1186 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex}) 1187 } else { 1188 r.logger.Debugf("%x [logterm: %d, index: %d] rejected msgApp [logterm: %d, index: %d] from %x", 1189 r.id, r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(m.Index)), m.Index, m.LogTerm, m.Index, m.From) 1190 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: m.Index, Reject: true, RejectHint: r.raftLog.lastIndex()}) 1191 } 1192 } 1193 1194 func (r *raft) handleHeartbeat(m pb.Message) { 1195 r.raftLog.commitTo(m.Commit) 1196 r.send(pb.Message{To: m.From, Type: pb.MsgHeartbeatResp, Context: m.Context}) 1197 } 1198 1199 func (r *raft) handleSnapshot(m pb.Message) { 1200 sindex, sterm := m.Snapshot.Metadata.Index, m.Snapshot.Metadata.Term 1201 if r.restore(m.Snapshot) { 1202 r.logger.Infof("%x [commit: %d] restored snapshot [index: %d, term: %d]", 1203 r.id, r.raftLog.committed, sindex, sterm) 1204 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.lastIndex()}) 1205 } else { 1206 r.logger.Infof("%x [commit: %d] ignored snapshot [index: %d, term: %d]", 1207 r.id, r.raftLog.committed, sindex, sterm) 1208 r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed}) 1209 } 1210 } 1211 1212 // restore recovers the state machine from a snapshot. It restores the log and the 1213 // configuration of state machine. 1214 func (r *raft) restore(s pb.Snapshot) bool { 1215 if s.Metadata.Index <= r.raftLog.committed { 1216 return false 1217 } 1218 if r.raftLog.matchTerm(s.Metadata.Index, s.Metadata.Term) { 1219 r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] fast-forwarded commit to snapshot [index: %d, term: %d]", 1220 r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term) 1221 r.raftLog.commitTo(s.Metadata.Index) 1222 return false 1223 } 1224 1225 // The normal peer can't become learner. 1226 if !r.isLearner { 1227 for _, id := range s.Metadata.ConfState.Learners { 1228 if id == r.id { 1229 r.logger.Errorf("%x can't become learner when restores snapshot [index: %d, term: %d]", r.id, s.Metadata.Index, s.Metadata.Term) 1230 return false 1231 } 1232 } 1233 } 1234 1235 r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] starts to restore snapshot [index: %d, term: %d]", 1236 r.id, r.raftLog.committed, r.raftLog.lastIndex(), r.raftLog.lastTerm(), s.Metadata.Index, s.Metadata.Term) 1237 1238 r.raftLog.restore(s) 1239 r.prs = make(map[uint64]*Progress) 1240 r.learnerPrs = make(map[uint64]*Progress) 1241 r.restoreNode(s.Metadata.ConfState.Nodes, false) 1242 r.restoreNode(s.Metadata.ConfState.Learners, true) 1243 return true 1244 } 1245 1246 func (r *raft) restoreNode(nodes []uint64, isLearner bool) { 1247 for _, n := range nodes { 1248 match, next := uint64(0), r.raftLog.lastIndex()+1 1249 if n == r.id { 1250 match = next - 1 1251 r.isLearner = isLearner 1252 } 1253 r.setProgress(n, match, next, isLearner) 1254 r.logger.Infof("%x restored progress of %x [%s]", r.id, n, r.getProgress(n)) 1255 } 1256 } 1257 1258 // promotable indicates whether state machine can be promoted to leader, 1259 // which is true when its own id is in progress list. 1260 func (r *raft) promotable() bool { 1261 _, ok := r.prs[r.id] 1262 return ok 1263 } 1264 1265 func (r *raft) addNode(id uint64) { 1266 r.addNodeOrLearnerNode(id, false) 1267 } 1268 1269 func (r *raft) addLearner(id uint64) { 1270 r.addNodeOrLearnerNode(id, true) 1271 } 1272 1273 func (r *raft) addNodeOrLearnerNode(id uint64, isLearner bool) { 1274 r.pendingConf = false 1275 pr := r.getProgress(id) 1276 if pr == nil { 1277 r.setProgress(id, 0, r.raftLog.lastIndex()+1, isLearner) 1278 } else { 1279 if isLearner && !pr.IsLearner { 1280 // can only change Learner to Voter 1281 r.logger.Infof("%x ignored addLeaner: do not support changing %x from raft peer to learner.", r.id, id) 1282 return 1283 } 1284 1285 if isLearner == pr.IsLearner { 1286 // Ignore any redundant addNode calls (which can happen because the 1287 // initial bootstrapping entries are applied twice). 1288 return 1289 } 1290 1291 // change Learner to Voter, use origin Learner progress 1292 delete(r.learnerPrs, id) 1293 pr.IsLearner = false 1294 r.prs[id] = pr 1295 } 1296 1297 if r.id == id { 1298 r.isLearner = isLearner 1299 } 1300 1301 // When a node is first added, we should mark it as recently active. 1302 // Otherwise, CheckQuorum may cause us to step down if it is invoked 1303 // before the added node has a chance to communicate with us. 1304 pr = r.getProgress(id) 1305 pr.RecentActive = true 1306 } 1307 1308 func (r *raft) removeNode(id uint64) { 1309 r.delProgress(id) 1310 r.pendingConf = false 1311 1312 // do not try to commit or abort transferring if there is no nodes in the cluster. 1313 if len(r.prs) == 0 && len(r.learnerPrs) == 0 { 1314 return 1315 } 1316 1317 // The quorum size is now smaller, so see if any pending entries can 1318 // be committed. 1319 if r.maybeCommit() { 1320 r.bcastAppend() 1321 } 1322 // If the removed node is the leadTransferee, then abort the leadership transferring. 1323 if r.state == StateLeader && r.leadTransferee == id { 1324 r.abortLeaderTransfer() 1325 } 1326 } 1327 1328 func (r *raft) resetPendingConf() { r.pendingConf = false } 1329 1330 func (r *raft) setProgress(id, match, next uint64, isLearner bool) { 1331 if !isLearner { 1332 delete(r.learnerPrs, id) 1333 r.prs[id] = &Progress{Next: next, Match: match, ins: newInflights(r.maxInflight)} 1334 return 1335 } 1336 1337 if _, ok := r.prs[id]; ok { 1338 panic(fmt.Sprintf("%x unexpected changing from voter to learner for %x", r.id, id)) 1339 } 1340 r.learnerPrs[id] = &Progress{Next: next, Match: match, ins: newInflights(r.maxInflight), IsLearner: true} 1341 } 1342 1343 func (r *raft) delProgress(id uint64) { 1344 delete(r.prs, id) 1345 delete(r.learnerPrs, id) 1346 } 1347 1348 func (r *raft) loadState(state pb.HardState) { 1349 if state.Commit < r.raftLog.committed || state.Commit > r.raftLog.lastIndex() { 1350 r.logger.Panicf("%x state.commit %d is out of range [%d, %d]", r.id, state.Commit, r.raftLog.committed, r.raftLog.lastIndex()) 1351 } 1352 r.raftLog.committed = state.Commit 1353 r.Term = state.Term 1354 r.Vote = state.Vote 1355 } 1356 1357 // pastElectionTimeout returns true iff r.electionElapsed is greater 1358 // than or equal to the randomized election timeout in 1359 // [electiontimeout, 2 * electiontimeout - 1]. 1360 func (r *raft) pastElectionTimeout() bool { 1361 return r.electionElapsed >= r.randomizedElectionTimeout 1362 } 1363 1364 func (r *raft) resetRandomizedElectionTimeout() { 1365 r.randomizedElectionTimeout = r.electionTimeout + globalRand.Intn(r.electionTimeout) 1366 } 1367 1368 // checkQuorumActive returns true if the quorum is active from 1369 // the view of the local raft state machine. Otherwise, it returns 1370 // false. 1371 // checkQuorumActive also resets all RecentActive to false. 1372 func (r *raft) checkQuorumActive() bool { 1373 var act int 1374 1375 r.forEachProgress(func(id uint64, pr *Progress) { 1376 if id == r.id { // self is always active 1377 act++ 1378 return 1379 } 1380 1381 if pr.RecentActive && !pr.IsLearner { 1382 act++ 1383 } 1384 1385 pr.RecentActive = false 1386 }) 1387 1388 return act >= r.quorum() 1389 } 1390 1391 func (r *raft) sendTimeoutNow(to uint64) { 1392 r.send(pb.Message{To: to, Type: pb.MsgTimeoutNow}) 1393 } 1394 1395 func (r *raft) abortLeaderTransfer() { 1396 r.leadTransferee = None 1397 } 1398 1399 func numOfPendingConf(ents []pb.Entry) int { 1400 n := 0 1401 for i := range ents { 1402 if ents[i].Type == pb.EntryConfChange { 1403 n++ 1404 } 1405 } 1406 return n 1407 }