github.com/djenriquez/nomad-1@v0.8.1/client/rpc.go (about) 1 package client 2 3 import ( 4 "errors" 5 "io" 6 "net" 7 "net/rpc" 8 "strings" 9 "time" 10 11 metrics "github.com/armon/go-metrics" 12 "github.com/hashicorp/consul/lib" 13 "github.com/hashicorp/nomad/client/servers" 14 inmem "github.com/hashicorp/nomad/helper/codec" 15 "github.com/hashicorp/nomad/helper/pool" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/yamux" 18 "github.com/ugorji/go/codec" 19 ) 20 21 // rpcEndpoints holds the RPC endpoints 22 type rpcEndpoints struct { 23 ClientStats *ClientStats 24 FileSystem *FileSystem 25 Allocations *Allocations 26 } 27 28 // ClientRPC is used to make a local, client only RPC call 29 func (c *Client) ClientRPC(method string, args interface{}, reply interface{}) error { 30 codec := &inmem.InmemCodec{ 31 Method: method, 32 Args: args, 33 Reply: reply, 34 } 35 if err := c.rpcServer.ServeRequest(codec); err != nil { 36 return err 37 } 38 return codec.Err 39 } 40 41 // StreamingRpcHandler is used to make a local, client only streaming RPC 42 // call. 43 func (c *Client) StreamingRpcHandler(method string) (structs.StreamingRpcHandler, error) { 44 return c.streamingRpcs.GetHandler(method) 45 } 46 47 // RPC is used to forward an RPC call to a nomad server, or fail if no servers. 48 func (c *Client) RPC(method string, args interface{}, reply interface{}) error { 49 // Invoke the RPCHandler if it exists 50 if c.config.RPCHandler != nil { 51 return c.config.RPCHandler.RPC(method, args, reply) 52 } 53 54 // This is subtle but we start measuring the time on the client side 55 // right at the time of the first request, vs. on the first retry as 56 // is done on the server side inside forward(). This is because the 57 // servers may already be applying the RPCHoldTimeout up there, so by 58 // starting the timer here we won't potentially double up the delay. 59 firstCheck := time.Now() 60 61 TRY: 62 server := c.servers.FindServer() 63 if server == nil { 64 return noServersErr 65 } 66 67 // Make the request. 68 rpcErr := c.connPool.RPC(c.Region(), server.Addr, c.RPCMajorVersion(), method, args, reply) 69 if rpcErr == nil { 70 c.fireRpcRetryWatcher() 71 return nil 72 } 73 74 // Move off to another server, and see if we can retry. 75 c.logger.Printf("[ERR] nomad: %q RPC failed to server %s: %v", method, server.Addr, rpcErr) 76 c.servers.NotifyFailedServer(server) 77 if retry := canRetry(args, rpcErr); !retry { 78 return rpcErr 79 } 80 81 // We can wait a bit and retry! 82 if time.Since(firstCheck) < c.config.RPCHoldTimeout { 83 jitter := lib.RandomStagger(c.config.RPCHoldTimeout / structs.JitterFraction) 84 select { 85 case <-time.After(jitter): 86 goto TRY 87 case <-c.shutdownCh: 88 } 89 } 90 return rpcErr 91 } 92 93 // canRetry returns true if the given situation is safe for a retry. 94 func canRetry(args interface{}, err error) bool { 95 // No leader errors are always safe to retry since no state could have 96 // been changed. 97 if structs.IsErrNoLeader(err) { 98 return true 99 } 100 101 // Reads are safe to retry for stream errors, such as if a server was 102 // being shut down. 103 info, ok := args.(structs.RPCInfo) 104 if ok && info.IsRead() && lib.IsErrEOF(err) { 105 return true 106 } 107 108 return false 109 } 110 111 // RemoteStreamingRpcHandler is used to make a streaming RPC call to a remote 112 // server. 113 func (c *Client) RemoteStreamingRpcHandler(method string) (structs.StreamingRpcHandler, error) { 114 server := c.servers.FindServer() 115 if server == nil { 116 return nil, noServersErr 117 } 118 119 conn, err := c.streamingRpcConn(server, method) 120 if err != nil { 121 // Move off to another server 122 c.logger.Printf("[ERR] nomad: %q RPC failed to server %s: %v", method, server.Addr, err) 123 c.servers.NotifyFailedServer(server) 124 return nil, err 125 } 126 127 return bridgedStreamingRpcHandler(conn), nil 128 } 129 130 // bridgedStreamingRpcHandler creates a bridged streaming RPC handler by copying 131 // data between the two sides. 132 func bridgedStreamingRpcHandler(sideA io.ReadWriteCloser) structs.StreamingRpcHandler { 133 return func(sideB io.ReadWriteCloser) { 134 defer sideA.Close() 135 defer sideB.Close() 136 structs.Bridge(sideA, sideB) 137 } 138 } 139 140 // streamingRpcConn is used to retrieve a connection to a server to conduct a 141 // streaming RPC. 142 func (c *Client) streamingRpcConn(server *servers.Server, method string) (net.Conn, error) { 143 // Dial the server 144 conn, err := net.DialTimeout("tcp", server.Addr.String(), 10*time.Second) 145 if err != nil { 146 return nil, err 147 } 148 149 // Cast to TCPConn 150 if tcp, ok := conn.(*net.TCPConn); ok { 151 tcp.SetKeepAlive(true) 152 tcp.SetNoDelay(true) 153 } 154 155 // Check if TLS is enabled 156 c.tlsWrapLock.RLock() 157 tlsWrap := c.tlsWrap 158 c.tlsWrapLock.RUnlock() 159 160 if tlsWrap != nil { 161 // Switch the connection into TLS mode 162 if _, err := conn.Write([]byte{byte(pool.RpcTLS)}); err != nil { 163 conn.Close() 164 return nil, err 165 } 166 167 // Wrap the connection in a TLS client 168 tlsConn, err := tlsWrap(c.Region(), conn) 169 if err != nil { 170 conn.Close() 171 return nil, err 172 } 173 conn = tlsConn 174 } 175 176 // Write the multiplex byte to set the mode 177 if _, err := conn.Write([]byte{byte(pool.RpcStreaming)}); err != nil { 178 conn.Close() 179 return nil, err 180 } 181 182 // Send the header 183 encoder := codec.NewEncoder(conn, structs.MsgpackHandle) 184 decoder := codec.NewDecoder(conn, structs.MsgpackHandle) 185 header := structs.StreamingRpcHeader{ 186 Method: method, 187 } 188 if err := encoder.Encode(header); err != nil { 189 conn.Close() 190 return nil, err 191 } 192 193 // Wait for the acknowledgement 194 var ack structs.StreamingRpcAck 195 if err := decoder.Decode(&ack); err != nil { 196 conn.Close() 197 return nil, err 198 } 199 200 if ack.Error != "" { 201 conn.Close() 202 return nil, errors.New(ack.Error) 203 } 204 205 return conn, nil 206 } 207 208 // setupClientRpc is used to setup the Client's RPC endpoints 209 func (c *Client) setupClientRpc() { 210 // Initialize the RPC handlers 211 c.endpoints.ClientStats = &ClientStats{c} 212 c.endpoints.FileSystem = NewFileSystemEndpoint(c) 213 c.endpoints.Allocations = &Allocations{c} 214 215 // Create the RPC Server 216 c.rpcServer = rpc.NewServer() 217 218 // Register the endpoints with the RPC server 219 c.setupClientRpcServer(c.rpcServer) 220 221 go c.rpcConnListener() 222 } 223 224 // setupClientRpcServer is used to populate a client RPC server with endpoints. 225 func (c *Client) setupClientRpcServer(server *rpc.Server) { 226 // Register the endpoints 227 server.Register(c.endpoints.ClientStats) 228 server.Register(c.endpoints.FileSystem) 229 server.Register(c.endpoints.Allocations) 230 } 231 232 // rpcConnListener is a long lived function that listens for new connections 233 // being made on the connection pool and starts an RPC listener for each 234 // connection. 235 func (c *Client) rpcConnListener() { 236 // Make a channel for new connections. 237 conns := make(chan *yamux.Session, 4) 238 c.connPool.SetConnListener(conns) 239 240 for { 241 select { 242 case <-c.shutdownCh: 243 return 244 case session, ok := <-conns: 245 if !ok { 246 continue 247 } 248 249 go c.listenConn(session) 250 } 251 } 252 } 253 254 // listenConn is used to listen for connections being made from the server on 255 // pre-existing connection. This should be called in a goroutine. 256 func (c *Client) listenConn(s *yamux.Session) { 257 for { 258 conn, err := s.Accept() 259 if err != nil { 260 if s.IsClosed() { 261 return 262 } 263 264 c.logger.Printf("[ERR] client.rpc: failed to accept RPC conn: %v", err) 265 continue 266 } 267 268 go c.handleConn(conn) 269 metrics.IncrCounter([]string{"client", "rpc", "accept_conn"}, 1) 270 } 271 } 272 273 // handleConn is used to determine if this is a RPC or Streaming RPC connection and 274 // invoke the correct handler 275 func (c *Client) handleConn(conn net.Conn) { 276 // Read a single byte 277 buf := make([]byte, 1) 278 if _, err := conn.Read(buf); err != nil { 279 if err != io.EOF { 280 c.logger.Printf("[ERR] client.rpc: failed to read byte: %v", err) 281 } 282 conn.Close() 283 return 284 } 285 286 // Switch on the byte 287 switch pool.RPCType(buf[0]) { 288 case pool.RpcNomad: 289 c.handleNomadConn(conn) 290 291 case pool.RpcStreaming: 292 c.handleStreamingConn(conn) 293 294 default: 295 c.logger.Printf("[ERR] client.rpc: unrecognized RPC byte: %v", buf[0]) 296 conn.Close() 297 return 298 } 299 } 300 301 // handleNomadConn is used to handle a single Nomad RPC connection. 302 func (c *Client) handleNomadConn(conn net.Conn) { 303 defer conn.Close() 304 rpcCodec := pool.NewServerCodec(conn) 305 for { 306 select { 307 case <-c.shutdownCh: 308 return 309 default: 310 } 311 312 if err := c.rpcServer.ServeRequest(rpcCodec); err != nil { 313 if err != io.EOF && !strings.Contains(err.Error(), "closed") { 314 c.logger.Printf("[ERR] client.rpc: RPC error: %v (%v)", err, conn) 315 metrics.IncrCounter([]string{"client", "rpc", "request_error"}, 1) 316 } 317 return 318 } 319 metrics.IncrCounter([]string{"client", "rpc", "request"}, 1) 320 } 321 } 322 323 // handleStreamingConn is used to handle a single Streaming Nomad RPC connection. 324 func (c *Client) handleStreamingConn(conn net.Conn) { 325 defer conn.Close() 326 327 // Decode the header 328 var header structs.StreamingRpcHeader 329 decoder := codec.NewDecoder(conn, structs.MsgpackHandle) 330 if err := decoder.Decode(&header); err != nil { 331 if err != io.EOF && !strings.Contains(err.Error(), "closed") { 332 c.logger.Printf("[ERR] client.rpc: Streaming RPC error: %v (%v)", err, conn) 333 metrics.IncrCounter([]string{"client", "streaming_rpc", "request_error"}, 1) 334 } 335 336 return 337 } 338 339 ack := structs.StreamingRpcAck{} 340 handler, err := c.streamingRpcs.GetHandler(header.Method) 341 if err != nil { 342 c.logger.Printf("[ERR] client.rpc: Streaming RPC error: %v (%v)", err, conn) 343 metrics.IncrCounter([]string{"client", "streaming_rpc", "request_error"}, 1) 344 ack.Error = err.Error() 345 } 346 347 // Send the acknowledgement 348 encoder := codec.NewEncoder(conn, structs.MsgpackHandle) 349 if err := encoder.Encode(ack); err != nil { 350 conn.Close() 351 return 352 } 353 354 if ack.Error != "" { 355 return 356 } 357 358 // Invoke the handler 359 metrics.IncrCounter([]string{"client", "streaming_rpc", "request"}, 1) 360 handler(conn) 361 } 362 363 // resolveServer given a sever's address as a string, return it's resolved 364 // net.Addr or an error. 365 func resolveServer(s string) (net.Addr, error) { 366 const defaultClientPort = "4647" // default client RPC port 367 host, port, err := net.SplitHostPort(s) 368 if err != nil { 369 if strings.Contains(err.Error(), "missing port") { 370 host = s 371 port = defaultClientPort 372 } else { 373 return nil, err 374 } 375 } 376 return net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) 377 } 378 379 // Ping is used to ping a particular server and returns whether it is healthy or 380 // a potential error. 381 func (c *Client) Ping(srv net.Addr) error { 382 var reply struct{} 383 err := c.connPool.RPC(c.Region(), srv, c.RPCMajorVersion(), "Status.Ping", struct{}{}, &reply) 384 return err 385 } 386 387 // rpcRetryWatcher returns a channel that will be closed if an event happens 388 // such that we expect the next RPC to be successful. 389 func (c *Client) rpcRetryWatcher() <-chan struct{} { 390 c.rpcRetryLock.Lock() 391 defer c.rpcRetryLock.Unlock() 392 393 if c.rpcRetryCh == nil { 394 c.rpcRetryCh = make(chan struct{}) 395 } 396 397 return c.rpcRetryCh 398 } 399 400 // fireRpcRetryWatcher causes any RPC retryloops to retry their RPCs because we 401 // believe the will be successful. 402 func (c *Client) fireRpcRetryWatcher() { 403 c.rpcRetryLock.Lock() 404 defer c.rpcRetryLock.Unlock() 405 if c.rpcRetryCh != nil { 406 close(c.rpcRetryCh) 407 c.rpcRetryCh = nil 408 } 409 }