github.com/DerekStrickland/consul@v1.4.5/agent/consul/server_serf.go (about) 1 package consul 2 3 import ( 4 "fmt" 5 "net" 6 "path/filepath" 7 "strings" 8 "time" 9 10 "github.com/hashicorp/consul/agent/metadata" 11 "github.com/hashicorp/consul/agent/structs" 12 "github.com/hashicorp/consul/lib" 13 "github.com/hashicorp/raft" 14 "github.com/hashicorp/serf/serf" 15 ) 16 17 const ( 18 // StatusReap is used to update the status of a node if we 19 // are handling a EventMemberReap 20 StatusReap = serf.MemberStatus(-1) 21 22 // userEventPrefix is pre-pended to a user event to distinguish it 23 userEventPrefix = "consul:event:" 24 25 // maxPeerRetries limits how many invalidate attempts are made 26 maxPeerRetries = 6 27 28 // peerRetryBase is a baseline retry time 29 peerRetryBase = 1 * time.Second 30 ) 31 32 // setupSerf is used to setup and initialize a Serf 33 func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, wan bool, wanPort int, 34 segment string, listener net.Listener) (*serf.Serf, error) { 35 conf.Init() 36 37 if wan { 38 conf.NodeName = fmt.Sprintf("%s.%s", s.config.NodeName, s.config.Datacenter) 39 } else { 40 conf.NodeName = s.config.NodeName 41 if wanPort > 0 { 42 conf.Tags["wan_join_port"] = fmt.Sprintf("%d", wanPort) 43 } 44 } 45 conf.Tags["role"] = "consul" 46 conf.Tags["dc"] = s.config.Datacenter 47 conf.Tags["segment"] = segment 48 if segment == "" { 49 for _, s := range s.config.Segments { 50 conf.Tags["sl_"+s.Name] = net.JoinHostPort(s.Advertise, fmt.Sprintf("%d", s.Port)) 51 } 52 } 53 conf.Tags["id"] = string(s.config.NodeID) 54 conf.Tags["vsn"] = fmt.Sprintf("%d", s.config.ProtocolVersion) 55 conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin) 56 conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax) 57 conf.Tags["raft_vsn"] = fmt.Sprintf("%d", s.config.RaftConfig.ProtocolVersion) 58 conf.Tags["build"] = s.config.Build 59 addr := listener.Addr().(*net.TCPAddr) 60 conf.Tags["port"] = fmt.Sprintf("%d", addr.Port) 61 if s.config.Bootstrap { 62 conf.Tags["bootstrap"] = "1" 63 } 64 if s.config.BootstrapExpect != 0 { 65 conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect) 66 } 67 if s.config.NonVoter { 68 conf.Tags["nonvoter"] = "1" 69 } 70 if s.config.UseTLS { 71 conf.Tags["use_tls"] = "1" 72 } 73 74 if s.acls.ACLsEnabled() { 75 // we start in legacy mode and allow upgrading later 76 conf.Tags["acls"] = string(structs.ACLModeLegacy) 77 } else { 78 conf.Tags["acls"] = string(structs.ACLModeDisabled) 79 } 80 if s.logger == nil { 81 conf.MemberlistConfig.LogOutput = s.config.LogOutput 82 conf.LogOutput = s.config.LogOutput 83 } 84 conf.MemberlistConfig.Logger = s.logger 85 conf.Logger = s.logger 86 conf.EventCh = ch 87 conf.ProtocolVersion = protocolVersionMap[s.config.ProtocolVersion] 88 conf.RejoinAfterLeave = s.config.RejoinAfterLeave 89 if wan { 90 conf.Merge = &wanMergeDelegate{} 91 } else { 92 conf.Merge = &lanMergeDelegate{ 93 dc: s.config.Datacenter, 94 nodeID: s.config.NodeID, 95 nodeName: s.config.NodeName, 96 segment: segment, 97 } 98 } 99 100 // Until Consul supports this fully, we disable automatic resolution. 101 // When enabled, the Serf gossip may just turn off if we are the minority 102 // node which is rather unexpected. 103 conf.EnableNameConflictResolution = false 104 105 if !s.config.DevMode { 106 conf.SnapshotPath = filepath.Join(s.config.DataDir, path) 107 } 108 if err := lib.EnsurePath(conf.SnapshotPath, false); err != nil { 109 return nil, err 110 } 111 112 return serf.Create(conf) 113 } 114 115 // userEventName computes the name of a user event 116 func userEventName(name string) string { 117 return userEventPrefix + name 118 } 119 120 // isUserEvent checks if a serf event is a user event 121 func isUserEvent(name string) bool { 122 return strings.HasPrefix(name, userEventPrefix) 123 } 124 125 // rawUserEventName is used to get the raw user event name 126 func rawUserEventName(name string) string { 127 return strings.TrimPrefix(name, userEventPrefix) 128 } 129 130 // lanEventHandler is used to handle events from the lan Serf cluster 131 func (s *Server) lanEventHandler() { 132 for { 133 select { 134 case e := <-s.eventChLAN: 135 switch e.EventType() { 136 case serf.EventMemberJoin: 137 s.lanNodeJoin(e.(serf.MemberEvent)) 138 s.localMemberEvent(e.(serf.MemberEvent)) 139 140 case serf.EventMemberLeave, serf.EventMemberFailed, serf.EventMemberReap: 141 s.lanNodeFailed(e.(serf.MemberEvent)) 142 s.localMemberEvent(e.(serf.MemberEvent)) 143 144 case serf.EventUser: 145 s.localEvent(e.(serf.UserEvent)) 146 case serf.EventMemberUpdate: 147 s.localMemberEvent(e.(serf.MemberEvent)) 148 case serf.EventQuery: // Ignore 149 default: 150 s.logger.Printf("[WARN] consul: Unhandled LAN Serf Event: %#v", e) 151 } 152 153 case <-s.shutdownCh: 154 return 155 } 156 } 157 } 158 159 // localMemberEvent is used to reconcile Serf events with the strongly 160 // consistent store if we are the current leader 161 func (s *Server) localMemberEvent(me serf.MemberEvent) { 162 // Do nothing if we are not the leader 163 if !s.IsLeader() { 164 return 165 } 166 167 // Check if this is a reap event 168 isReap := me.EventType() == serf.EventMemberReap 169 170 // Queue the members for reconciliation 171 for _, m := range me.Members { 172 // Change the status if this is a reap event 173 if isReap { 174 m.Status = StatusReap 175 } 176 select { 177 case s.reconcileCh <- m: 178 default: 179 } 180 } 181 } 182 183 // localEvent is called when we receive an event on the local Serf 184 func (s *Server) localEvent(event serf.UserEvent) { 185 // Handle only consul events 186 if !strings.HasPrefix(event.Name, "consul:") { 187 return 188 } 189 190 switch name := event.Name; { 191 case name == newLeaderEvent: 192 s.logger.Printf("[INFO] consul: New leader elected: %s", event.Payload) 193 194 // Trigger the callback 195 if s.config.ServerUp != nil { 196 s.config.ServerUp() 197 } 198 case isUserEvent(name): 199 event.Name = rawUserEventName(name) 200 s.logger.Printf("[DEBUG] consul: User event: %s", event.Name) 201 202 // Trigger the callback 203 if s.config.UserEventHandler != nil { 204 s.config.UserEventHandler(event) 205 } 206 default: 207 if !s.handleEnterpriseUserEvents(event) { 208 s.logger.Printf("[WARN] consul: Unhandled local event: %v", event) 209 } 210 } 211 } 212 213 // lanNodeJoin is used to handle join events on the LAN pool. 214 func (s *Server) lanNodeJoin(me serf.MemberEvent) { 215 for _, m := range me.Members { 216 ok, serverMeta := metadata.IsConsulServer(m) 217 if !ok || serverMeta.Segment != "" { 218 continue 219 } 220 s.logger.Printf("[INFO] consul: Adding LAN server %s", serverMeta) 221 222 // Update server lookup 223 s.serverLookup.AddServer(serverMeta) 224 225 // If we're still expecting to bootstrap, may need to handle this. 226 if s.config.BootstrapExpect != 0 { 227 s.maybeBootstrap() 228 } 229 230 // Kick the join flooders. 231 s.FloodNotify() 232 } 233 } 234 235 // maybeBootstrap is used to handle bootstrapping when a new consul server joins. 236 func (s *Server) maybeBootstrap() { 237 // Bootstrap can only be done if there are no committed logs, remove our 238 // expectations of bootstrapping. This is slightly cheaper than the full 239 // check that BootstrapCluster will do, so this is a good pre-filter. 240 index, err := s.raftStore.LastIndex() 241 if err != nil { 242 s.logger.Printf("[ERR] consul: Failed to read last raft index: %v", err) 243 return 244 } 245 if index != 0 { 246 s.logger.Printf("[INFO] consul: Raft data found, disabling bootstrap mode") 247 s.config.BootstrapExpect = 0 248 return 249 } 250 251 // Scan for all the known servers. 252 members := s.serfLAN.Members() 253 var servers []metadata.Server 254 voters := 0 255 for _, member := range members { 256 valid, p := metadata.IsConsulServer(member) 257 if !valid { 258 continue 259 } 260 if p.Datacenter != s.config.Datacenter { 261 s.logger.Printf("[ERR] consul: Member %v has a conflicting datacenter, ignoring", member) 262 continue 263 } 264 if p.Expect != 0 && p.Expect != s.config.BootstrapExpect { 265 s.logger.Printf("[ERR] consul: Member %v has a conflicting expect value. All nodes should expect the same number.", member) 266 return 267 } 268 if p.Bootstrap { 269 s.logger.Printf("[ERR] consul: Member %v has bootstrap mode. Expect disabled.", member) 270 return 271 } 272 if !p.NonVoter { 273 voters++ 274 } 275 servers = append(servers, *p) 276 } 277 278 // Skip if we haven't met the minimum expect count. 279 if voters < s.config.BootstrapExpect { 280 return 281 } 282 283 // Query each of the servers and make sure they report no Raft peers. 284 for _, server := range servers { 285 var peers []string 286 287 // Retry with exponential backoff to get peer status from this server 288 for attempt := uint(0); attempt < maxPeerRetries; attempt++ { 289 if err := s.connPool.RPC(s.config.Datacenter, server.Addr, server.Version, 290 "Status.Peers", server.UseTLS, &struct{}{}, &peers); err != nil { 291 nextRetry := time.Duration((1 << attempt) * peerRetryBase) 292 s.logger.Printf("[ERR] consul: Failed to confirm peer status for %s: %v. Retrying in "+ 293 "%v...", server.Name, err, nextRetry.String()) 294 time.Sleep(nextRetry) 295 } else { 296 break 297 } 298 } 299 300 // Found a node with some Raft peers, stop bootstrap since there's 301 // evidence of an existing cluster. We should get folded in by the 302 // existing servers if that's the case, so it's cleaner to sit as a 303 // candidate with no peers so we don't cause spurious elections. 304 // It's OK this is racy, because even with an initial bootstrap 305 // as long as one peer runs bootstrap things will work, and if we 306 // have multiple peers bootstrap in the same way, that's OK. We 307 // just don't want a server added much later to do a live bootstrap 308 // and interfere with the cluster. This isn't required for Raft's 309 // correctness because no server in the existing cluster will vote 310 // for this server, but it makes things much more stable. 311 if len(peers) > 0 { 312 s.logger.Printf("[INFO] consul: Existing Raft peers reported by %s, disabling bootstrap mode", server.Name) 313 s.config.BootstrapExpect = 0 314 return 315 } 316 } 317 318 // Attempt a live bootstrap! 319 var configuration raft.Configuration 320 var addrs []string 321 minRaftVersion, err := s.autopilot.MinRaftProtocol() 322 if err != nil { 323 s.logger.Printf("[ERR] consul: Failed to read server raft versions: %v", err) 324 } 325 326 for _, server := range servers { 327 addr := server.Addr.String() 328 addrs = append(addrs, addr) 329 var id raft.ServerID 330 if minRaftVersion >= 3 { 331 id = raft.ServerID(server.ID) 332 } else { 333 id = raft.ServerID(addr) 334 } 335 suffrage := raft.Voter 336 if server.NonVoter { 337 suffrage = raft.Nonvoter 338 } 339 peer := raft.Server{ 340 ID: id, 341 Address: raft.ServerAddress(addr), 342 Suffrage: suffrage, 343 } 344 configuration.Servers = append(configuration.Servers, peer) 345 } 346 s.logger.Printf("[INFO] consul: Found expected number of peers, attempting bootstrap: %s", 347 strings.Join(addrs, ",")) 348 future := s.raft.BootstrapCluster(configuration) 349 if err := future.Error(); err != nil { 350 s.logger.Printf("[ERR] consul: Failed to bootstrap cluster: %v", err) 351 } 352 353 // Bootstrapping complete, or failed for some reason, don't enter this 354 // again. 355 s.config.BootstrapExpect = 0 356 } 357 358 // lanNodeFailed is used to handle fail events on the LAN pool. 359 func (s *Server) lanNodeFailed(me serf.MemberEvent) { 360 for _, m := range me.Members { 361 ok, serverMeta := metadata.IsConsulServer(m) 362 if !ok || serverMeta.Segment != "" { 363 continue 364 } 365 s.logger.Printf("[INFO] consul: Removing LAN server %s", serverMeta) 366 367 // Update id to address map 368 s.serverLookup.RemoveServer(serverMeta) 369 } 370 }