github.com/kafkaliu/etcd@v0.1.2-0.20131007164923-44c16dd30d69/raft_server.go (about) 1 /* 2 Copyright 2013 CoreOS Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "crypto/tls" 22 "encoding/binary" 23 "encoding/json" 24 "fmt" 25 "io/ioutil" 26 "net/http" 27 "net/url" 28 "time" 29 30 etcdErr "github.com/coreos/etcd/error" 31 "github.com/coreos/go-raft" 32 ) 33 34 type raftServer struct { 35 *raft.Server 36 version string 37 joinIndex uint64 38 name string 39 url string 40 listenHost string 41 tlsConf *TLSConfig 42 tlsInfo *TLSInfo 43 followersStats *raftFollowersStats 44 serverStats *raftServerStats 45 } 46 47 var r *raftServer 48 49 func newRaftServer(name string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo) *raftServer { 50 51 // Create transporter for raft 52 raftTransporter := newTransporter(tlsConf.Scheme, tlsConf.Client) 53 54 // Create raft server 55 server, err := raft.NewServer(name, dirPath, raftTransporter, etcdStore, nil, "") 56 57 check(err) 58 59 return &raftServer{ 60 Server: server, 61 version: raftVersion, 62 name: name, 63 url: url, 64 listenHost: listenHost, 65 tlsConf: tlsConf, 66 tlsInfo: tlsInfo, 67 followersStats: &raftFollowersStats{ 68 Leader: name, 69 Followers: make(map[string]*raftFollowerStats), 70 }, 71 serverStats: &raftServerStats{ 72 StartTime: time.Now(), 73 sendRateQueue: &statsQueue{ 74 back: -1, 75 }, 76 recvRateQueue: &statsQueue{ 77 back: -1, 78 }, 79 }, 80 } 81 } 82 83 // Start the raft server 84 func (r *raftServer) ListenAndServe() { 85 // Setup commands. 86 registerCommands() 87 88 // LoadSnapshot 89 if snapshot { 90 err := r.LoadSnapshot() 91 92 if err == nil { 93 debugf("%s finished load snapshot", r.name) 94 } else { 95 debug(err) 96 } 97 } 98 99 r.SetElectionTimeout(ElectionTimeout) 100 r.SetHeartbeatTimeout(HeartbeatTimeout) 101 102 r.Start() 103 104 if r.IsLogEmpty() { 105 106 // start as a leader in a new cluster 107 if len(cluster) == 0 { 108 startAsLeader() 109 110 } else { 111 startAsFollower() 112 } 113 114 } else { 115 116 // rejoin the previous cluster 117 cluster = getMachines(nameToRaftURL) 118 for i := 0; i < len(cluster); i++ { 119 u, err := url.Parse(cluster[i]) 120 if err != nil { 121 debug("rejoin cannot parse url: ", err) 122 } 123 cluster[i] = u.Host 124 } 125 ok := joinCluster(cluster) 126 if !ok { 127 warn("the entire cluster is down! this machine will restart the cluster.") 128 } 129 130 debugf("%s restart as a follower", r.name) 131 } 132 133 // open the snapshot 134 if snapshot { 135 go monitorSnapshot() 136 } 137 138 // start to response to raft requests 139 go r.startTransport(r.tlsConf.Scheme, r.tlsConf.Server) 140 141 } 142 143 func startAsLeader() { 144 // leader need to join self as a peer 145 for { 146 _, err := r.Do(newJoinCommand()) 147 if err == nil { 148 break 149 } 150 } 151 debugf("%s start as a leader", r.name) 152 } 153 154 func startAsFollower() { 155 // start as a follower in a existing cluster 156 for i := 0; i < retryTimes; i++ { 157 ok := joinCluster(cluster) 158 if ok { 159 return 160 } 161 warnf("cannot join to cluster via given machines, retry in %d seconds", RetryInterval) 162 time.Sleep(time.Second * RetryInterval) 163 } 164 165 fatalf("Cannot join the cluster via given machines after %x retries", retryTimes) 166 } 167 168 // Start to listen and response raft command 169 func (r *raftServer) startTransport(scheme string, tlsConf tls.Config) { 170 infof("raft server [name %s, listen on %s, advertised url %s]", r.name, r.listenHost, r.url) 171 172 raftMux := http.NewServeMux() 173 174 server := &http.Server{ 175 Handler: raftMux, 176 TLSConfig: &tlsConf, 177 Addr: r.listenHost, 178 } 179 180 // internal commands 181 raftMux.HandleFunc("/name", NameHttpHandler) 182 raftMux.HandleFunc("/version", RaftVersionHttpHandler) 183 raftMux.Handle("/join", errorHandler(JoinHttpHandler)) 184 raftMux.HandleFunc("/remove/", RemoveHttpHandler) 185 raftMux.HandleFunc("/vote", VoteHttpHandler) 186 raftMux.HandleFunc("/log", GetLogHttpHandler) 187 raftMux.HandleFunc("/log/append", AppendEntriesHttpHandler) 188 raftMux.HandleFunc("/snapshot", SnapshotHttpHandler) 189 raftMux.HandleFunc("/snapshotRecovery", SnapshotRecoveryHttpHandler) 190 raftMux.HandleFunc("/etcdURL", EtcdURLHttpHandler) 191 192 if scheme == "http" { 193 fatal(server.ListenAndServe()) 194 } else { 195 fatal(server.ListenAndServeTLS(r.tlsInfo.CertFile, r.tlsInfo.KeyFile)) 196 } 197 198 } 199 200 // getVersion fetches the raft version of a peer. This works for now but we 201 // will need to do something more sophisticated later when we allow mixed 202 // version clusters. 203 func getVersion(t *transporter, versionURL url.URL) (string, error) { 204 resp, req, err := t.Get(versionURL.String()) 205 206 if err != nil { 207 return "", err 208 } 209 210 defer resp.Body.Close() 211 212 t.CancelWhenTimeout(req) 213 214 body, err := ioutil.ReadAll(resp.Body) 215 216 return string(body), nil 217 } 218 219 func joinCluster(cluster []string) bool { 220 for _, machine := range cluster { 221 222 if len(machine) == 0 { 223 continue 224 } 225 226 err := joinByMachine(r.Server, machine, r.tlsConf.Scheme) 227 if err == nil { 228 debugf("%s success join to the cluster via machine %s", r.name, machine) 229 return true 230 231 } else { 232 if _, ok := err.(etcdErr.Error); ok { 233 fatal(err) 234 } 235 236 debugf("cannot join to cluster via machine %s %s", machine, err) 237 } 238 } 239 return false 240 } 241 242 // Send join requests to machine. 243 func joinByMachine(s *raft.Server, machine string, scheme string) error { 244 var b bytes.Buffer 245 246 // t must be ok 247 t, _ := r.Transporter().(*transporter) 248 249 // Our version must match the leaders version 250 versionURL := url.URL{Host: machine, Scheme: scheme, Path: "/version"} 251 version, err := getVersion(t, versionURL) 252 if err != nil { 253 return fmt.Errorf("Unable to join: %v", err) 254 } 255 256 // TODO: versioning of the internal protocol. See: 257 // Documentation/internatl-protocol-versioning.md 258 if version != r.version { 259 return fmt.Errorf("Unable to join: internal version mismatch, entire cluster must be running identical versions of etcd") 260 } 261 262 json.NewEncoder(&b).Encode(newJoinCommand()) 263 264 joinURL := url.URL{Host: machine, Scheme: scheme, Path: "/join"} 265 266 debugf("Send Join Request to %s", joinURL.String()) 267 268 resp, req, err := t.Post(joinURL.String(), &b) 269 270 for { 271 if err != nil { 272 return fmt.Errorf("Unable to join: %v", err) 273 } 274 if resp != nil { 275 defer resp.Body.Close() 276 277 t.CancelWhenTimeout(req) 278 279 if resp.StatusCode == http.StatusOK { 280 b, _ := ioutil.ReadAll(resp.Body) 281 r.joinIndex, _ = binary.Uvarint(b) 282 return nil 283 } 284 if resp.StatusCode == http.StatusTemporaryRedirect { 285 286 address := resp.Header.Get("Location") 287 debugf("Send Join Request to %s", address) 288 289 json.NewEncoder(&b).Encode(newJoinCommand()) 290 291 resp, req, err = t.Post(address, &b) 292 293 } else if resp.StatusCode == http.StatusBadRequest { 294 debug("Reach max number machines in the cluster") 295 decoder := json.NewDecoder(resp.Body) 296 err := &etcdErr.Error{} 297 decoder.Decode(err) 298 return *err 299 } else { 300 return fmt.Errorf("Unable to join") 301 } 302 } 303 304 } 305 return fmt.Errorf("Unable to join: %v", err) 306 } 307 308 func (r *raftServer) Stats() []byte { 309 r.serverStats.LeaderInfo.Uptime = time.Now().Sub(r.serverStats.LeaderInfo.startTime).String() 310 311 queue := r.serverStats.sendRateQueue 312 313 r.serverStats.SendingPkgRate, r.serverStats.SendingBandwidthRate = queue.Rate() 314 315 queue = r.serverStats.recvRateQueue 316 317 r.serverStats.RecvingPkgRate, r.serverStats.RecvingBandwidthRate = queue.Rate() 318 319 b, _ := json.Marshal(r.serverStats) 320 321 return b 322 } 323 324 func (r *raftServer) PeerStats() []byte { 325 if r.State() == raft.Leader { 326 b, _ := json.Marshal(r.followersStats) 327 return b 328 } 329 return nil 330 } 331 332 // Register commands to raft server 333 func registerCommands() { 334 raft.RegisterCommand(&JoinCommand{}) 335 raft.RegisterCommand(&RemoveCommand{}) 336 raft.RegisterCommand(&SetCommand{}) 337 raft.RegisterCommand(&GetCommand{}) 338 raft.RegisterCommand(&DeleteCommand{}) 339 raft.RegisterCommand(&WatchCommand{}) 340 raft.RegisterCommand(&TestAndSetCommand{}) 341 }