github.com/huiliang/nomad@v0.2.1-0.20151124023127-7a8b664699ff/nomad/rpc.go (about) 1 package nomad 2 3 import ( 4 "crypto/tls" 5 "fmt" 6 "io" 7 "math/rand" 8 "net" 9 "net/rpc" 10 "strings" 11 "time" 12 13 "github.com/armon/go-metrics" 14 "github.com/hashicorp/net-rpc-msgpackrpc" 15 "github.com/hashicorp/nomad/nomad/state" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/nomad/nomad/watch" 18 "github.com/hashicorp/raft" 19 "github.com/hashicorp/yamux" 20 ) 21 22 type RPCType byte 23 24 const ( 25 rpcNomad RPCType = 0x01 26 rpcRaft = 0x02 27 rpcMultiplex = 0x03 28 rpcTLS = 0x04 29 ) 30 31 const ( 32 // maxQueryTime is used to bound the limit of a blocking query 33 maxQueryTime = 300 * time.Second 34 35 // defaultQueryTime is the amount of time we block waiting for a change 36 // if no time is specified. Previously we would wait the maxQueryTime. 37 defaultQueryTime = 300 * time.Second 38 39 // jitterFraction is a the limit to the amount of jitter we apply 40 // to a user specified MaxQueryTime. We divide the specified time by 41 // the fraction. So 16 == 6.25% limit of jitter 42 jitterFraction = 16 43 44 // Warn if the Raft command is larger than this. 45 // If it's over 1MB something is probably being abusive. 46 raftWarnSize = 1024 * 1024 47 48 // enqueueLimit caps how long we will wait to enqueue 49 // a new Raft command. Something is probably wrong if this 50 // value is ever reached. However, it prevents us from blocking 51 // the requesting goroutine forever. 52 enqueueLimit = 30 * time.Second 53 ) 54 55 // NewClientCodec returns a new rpc.ClientCodec to be used to make RPC calls to 56 // the Nomad Server. 57 func NewClientCodec(conn io.ReadWriteCloser) rpc.ClientCodec { 58 return msgpackrpc.NewCodecFromHandle(true, true, conn, structs.MsgpackHandle) 59 } 60 61 // NewServerCodec returns a new rpc.ServerCodec to be used by the Nomad Server 62 // to handle rpcs. 63 func NewServerCodec(conn io.ReadWriteCloser) rpc.ServerCodec { 64 return msgpackrpc.NewCodecFromHandle(true, true, conn, structs.MsgpackHandle) 65 } 66 67 // listen is used to listen for incoming RPC connections 68 func (s *Server) listen() { 69 for { 70 // Accept a connection 71 conn, err := s.rpcListener.Accept() 72 if err != nil { 73 if s.shutdown { 74 return 75 } 76 s.logger.Printf("[ERR] nomad.rpc: failed to accept RPC conn: %v", err) 77 continue 78 } 79 80 go s.handleConn(conn, false) 81 metrics.IncrCounter([]string{"nomad", "rpc", "accept_conn"}, 1) 82 } 83 } 84 85 // handleConn is used to determine if this is a Raft or 86 // Nomad type RPC connection and invoke the correct handler 87 func (s *Server) handleConn(conn net.Conn, isTLS bool) { 88 // Read a single byte 89 buf := make([]byte, 1) 90 if _, err := conn.Read(buf); err != nil { 91 if err != io.EOF { 92 s.logger.Printf("[ERR] nomad.rpc: failed to read byte: %v", err) 93 } 94 conn.Close() 95 return 96 } 97 98 // Enforce TLS if VerifyIncoming is set 99 if s.config.RequireTLS && !isTLS && RPCType(buf[0]) != rpcTLS { 100 s.logger.Printf("[WARN] nomad.rpc: Non-TLS connection attempted with RequireTLS set") 101 conn.Close() 102 return 103 } 104 105 // Switch on the byte 106 switch RPCType(buf[0]) { 107 case rpcNomad: 108 s.handleNomadConn(conn) 109 110 case rpcRaft: 111 metrics.IncrCounter([]string{"nomad", "rpc", "raft_handoff"}, 1) 112 s.raftLayer.Handoff(conn) 113 114 case rpcMultiplex: 115 s.handleMultiplex(conn) 116 117 case rpcTLS: 118 if s.rpcTLS == nil { 119 s.logger.Printf("[WARN] nomad.rpc: TLS connection attempted, server not configured for TLS") 120 conn.Close() 121 return 122 } 123 conn = tls.Server(conn, s.rpcTLS) 124 s.handleConn(conn, true) 125 126 default: 127 s.logger.Printf("[ERR] nomad.rpc: unrecognized RPC byte: %v", buf[0]) 128 conn.Close() 129 return 130 } 131 } 132 133 // handleMultiplex is used to multiplex a single incoming connection 134 // using the Yamux multiplexer 135 func (s *Server) handleMultiplex(conn net.Conn) { 136 defer conn.Close() 137 conf := yamux.DefaultConfig() 138 conf.LogOutput = s.config.LogOutput 139 server, _ := yamux.Server(conn, conf) 140 for { 141 sub, err := server.Accept() 142 if err != nil { 143 if err != io.EOF { 144 s.logger.Printf("[ERR] nomad.rpc: multiplex conn accept failed: %v", err) 145 } 146 return 147 } 148 go s.handleNomadConn(sub) 149 } 150 } 151 152 // handleNomadConn is used to service a single Nomad RPC connection 153 func (s *Server) handleNomadConn(conn net.Conn) { 154 defer conn.Close() 155 rpcCodec := NewServerCodec(conn) 156 for { 157 select { 158 case <-s.shutdownCh: 159 return 160 default: 161 } 162 163 if err := s.rpcServer.ServeRequest(rpcCodec); err != nil { 164 if err != io.EOF && !strings.Contains(err.Error(), "closed") { 165 s.logger.Printf("[ERR] nomad.rpc: RPC error: %v (%v)", err, conn) 166 metrics.IncrCounter([]string{"nomad", "rpc", "request_error"}, 1) 167 } 168 return 169 } 170 metrics.IncrCounter([]string{"nomad", "rpc", "request"}, 1) 171 } 172 } 173 174 // forward is used to forward to a remote region or to forward to the local leader 175 // Returns a bool of if forwarding was performed, as well as any error 176 func (s *Server) forward(method string, info structs.RPCInfo, args interface{}, reply interface{}) (bool, error) { 177 region := info.RequestRegion() 178 if region == "" { 179 return true, fmt.Errorf("missing target RPC") 180 } 181 182 // Handle region forwarding 183 if region != s.config.Region { 184 err := s.forwardRegion(region, method, args, reply) 185 return true, err 186 } 187 188 // Check if we can allow a stale read 189 if info.IsRead() && info.AllowStaleRead() { 190 return false, nil 191 } 192 193 // Handle leader forwarding 194 if !s.IsLeader() { 195 err := s.forwardLeader(method, args, reply) 196 return true, err 197 } 198 return false, nil 199 } 200 201 // forwardLeader is used to forward an RPC call to the leader, or fail if no leader 202 func (s *Server) forwardLeader(method string, args interface{}, reply interface{}) error { 203 // Get the leader 204 leader := s.raft.Leader() 205 if leader == "" { 206 return structs.ErrNoLeader 207 } 208 209 // Lookup the server 210 s.peerLock.RLock() 211 server := s.localPeers[leader] 212 s.peerLock.RUnlock() 213 214 // Handle a missing server 215 if server == nil { 216 return structs.ErrNoLeader 217 } 218 return s.connPool.RPC(s.config.Region, server.Addr, server.Version, method, args, reply) 219 } 220 221 // forwardRegion is used to forward an RPC call to a remote region, or fail if no servers 222 func (s *Server) forwardRegion(region, method string, args interface{}, reply interface{}) error { 223 // Bail if we can't find any servers 224 s.peerLock.RLock() 225 servers := s.peers[region] 226 if len(servers) == 0 { 227 s.peerLock.RUnlock() 228 s.logger.Printf("[WARN] nomad.rpc: RPC request for region '%s', no path found", 229 region) 230 return structs.ErrNoRegionPath 231 } 232 233 // Select a random addr 234 offset := rand.Int31() % int32(len(servers)) 235 server := servers[offset] 236 s.peerLock.RUnlock() 237 238 // Forward to remote Nomad 239 metrics.IncrCounter([]string{"nomad", "rpc", "cross-region", region}, 1) 240 return s.connPool.RPC(region, server.Addr, server.Version, method, args, reply) 241 } 242 243 // raftApplyFuture is used to encode a message, run it through raft, and return the Raft future. 244 func (s *Server) raftApplyFuture(t structs.MessageType, msg interface{}) (raft.ApplyFuture, error) { 245 buf, err := structs.Encode(t, msg) 246 if err != nil { 247 return nil, fmt.Errorf("Failed to encode request: %v", err) 248 } 249 250 // Warn if the command is very large 251 if n := len(buf); n > raftWarnSize { 252 s.logger.Printf("[WARN] nomad: Attempting to apply large raft entry (type %d) (%d bytes)", t, n) 253 } 254 255 future := s.raft.Apply(buf, enqueueLimit) 256 return future, nil 257 } 258 259 // raftApply is used to encode a message, run it through raft, and return 260 // the FSM response along with any errors 261 func (s *Server) raftApply(t structs.MessageType, msg interface{}) (interface{}, uint64, error) { 262 future, err := s.raftApplyFuture(t, msg) 263 if err != nil { 264 return nil, 0, err 265 } 266 if err := future.Error(); err != nil { 267 return nil, 0, err 268 } 269 return future.Response(), future.Index(), nil 270 } 271 272 // setQueryMeta is used to populate the QueryMeta data for an RPC call 273 func (s *Server) setQueryMeta(m *structs.QueryMeta) { 274 if s.IsLeader() { 275 m.LastContact = 0 276 m.KnownLeader = true 277 } else { 278 m.LastContact = time.Now().Sub(s.raft.LastContact()) 279 m.KnownLeader = (s.raft.Leader() != "") 280 } 281 } 282 283 // blockingOptions is used to parameterize blockingRPC 284 type blockingOptions struct { 285 queryOpts *structs.QueryOptions 286 queryMeta *structs.QueryMeta 287 watch watch.Items 288 run func() error 289 } 290 291 // blockingRPC is used for queries that need to wait for a 292 // minimum index. This is used to block and wait for changes. 293 func (s *Server) blockingRPC(opts *blockingOptions) error { 294 var timeout *time.Timer 295 var notifyCh chan struct{} 296 var state *state.StateStore 297 298 // Fast path non-blocking 299 if opts.queryOpts.MinQueryIndex == 0 { 300 goto RUN_QUERY 301 } 302 303 // Restrict the max query time, and ensure there is always one 304 if opts.queryOpts.MaxQueryTime > maxQueryTime { 305 opts.queryOpts.MaxQueryTime = maxQueryTime 306 } else if opts.queryOpts.MaxQueryTime <= 0 { 307 opts.queryOpts.MaxQueryTime = defaultQueryTime 308 } 309 310 // Apply a small amount of jitter to the request 311 opts.queryOpts.MaxQueryTime += randomStagger(opts.queryOpts.MaxQueryTime / jitterFraction) 312 313 // Setup a query timeout 314 timeout = time.NewTimer(opts.queryOpts.MaxQueryTime) 315 316 // Setup the notify channel 317 notifyCh = make(chan struct{}, 1) 318 319 // Ensure we tear down any watchers on return 320 state = s.fsm.State() 321 defer func() { 322 timeout.Stop() 323 state.StopWatch(opts.watch, notifyCh) 324 }() 325 326 REGISTER_NOTIFY: 327 // Register the notification channel. This may be done 328 // multiple times if we have not reached the target wait index. 329 state.Watch(opts.watch, notifyCh) 330 331 RUN_QUERY: 332 // Update the query meta data 333 s.setQueryMeta(opts.queryMeta) 334 335 // Run the query function 336 metrics.IncrCounter([]string{"nomad", "rpc", "query"}, 1) 337 err := opts.run() 338 339 // Check for minimum query time 340 if err == nil && opts.queryOpts.MinQueryIndex > 0 && opts.queryMeta.Index <= opts.queryOpts.MinQueryIndex { 341 select { 342 case <-notifyCh: 343 goto REGISTER_NOTIFY 344 case <-timeout.C: 345 } 346 } 347 return err 348 }