github.com/manicqin/nomad@v0.9.5/client/rpc.go (about) 1 package client 2 3 import ( 4 "errors" 5 "io" 6 "net" 7 "net/rpc" 8 "strings" 9 "time" 10 11 metrics "github.com/armon/go-metrics" 12 "github.com/hashicorp/consul/lib" 13 "github.com/hashicorp/nomad/client/servers" 14 inmem "github.com/hashicorp/nomad/helper/codec" 15 "github.com/hashicorp/nomad/helper/pool" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/yamux" 18 "github.com/ugorji/go/codec" 19 ) 20 21 // rpcEndpoints holds the RPC endpoints 22 type rpcEndpoints struct { 23 ClientStats *ClientStats 24 FileSystem *FileSystem 25 Allocations *Allocations 26 Agent *Agent 27 } 28 29 // ClientRPC is used to make a local, client only RPC call 30 func (c *Client) ClientRPC(method string, args interface{}, reply interface{}) error { 31 codec := &inmem.InmemCodec{ 32 Method: method, 33 Args: args, 34 Reply: reply, 35 } 36 if err := c.rpcServer.ServeRequest(codec); err != nil { 37 return err 38 } 39 return codec.Err 40 } 41 42 // StreamingRpcHandler is used to make a local, client only streaming RPC 43 // call. 44 func (c *Client) StreamingRpcHandler(method string) (structs.StreamingRpcHandler, error) { 45 return c.streamingRpcs.GetHandler(method) 46 } 47 48 // RPC is used to forward an RPC call to a nomad server, or fail if no servers. 49 func (c *Client) RPC(method string, args interface{}, reply interface{}) error { 50 // Invoke the RPCHandler if it exists 51 if c.config.RPCHandler != nil { 52 return c.config.RPCHandler.RPC(method, args, reply) 53 } 54 55 // This is subtle but we start measuring the time on the client side 56 // right at the time of the first request, vs. on the first retry as 57 // is done on the server side inside forward(). This is because the 58 // servers may already be applying the RPCHoldTimeout up there, so by 59 // starting the timer here we won't potentially double up the delay. 60 firstCheck := time.Now() 61 62 TRY: 63 server := c.servers.FindServer() 64 if server == nil { 65 return noServersErr 66 } 67 68 // Make the request. 69 rpcErr := c.connPool.RPC(c.Region(), server.Addr, c.RPCMajorVersion(), method, args, reply) 70 if rpcErr == nil { 71 c.fireRpcRetryWatcher() 72 return nil 73 } 74 75 // If shutting down, exit without logging the error 76 select { 77 case <-c.shutdownCh: 78 return nil 79 default: 80 } 81 82 // Move off to another server, and see if we can retry. 83 c.rpcLogger.Error("error performing RPC to server", "error", rpcErr, "rpc", method, "server", server.Addr) 84 c.servers.NotifyFailedServer(server) 85 if retry := canRetry(args, rpcErr); !retry { 86 return rpcErr 87 } 88 89 // We can wait a bit and retry! 90 if time.Since(firstCheck) < c.config.RPCHoldTimeout { 91 jitter := lib.RandomStagger(c.config.RPCHoldTimeout / structs.JitterFraction) 92 select { 93 case <-time.After(jitter): 94 goto TRY 95 case <-c.shutdownCh: 96 } 97 } 98 return rpcErr 99 } 100 101 // canRetry returns true if the given situation is safe for a retry. 102 func canRetry(args interface{}, err error) bool { 103 // No leader errors are always safe to retry since no state could have 104 // been changed. 105 if structs.IsErrNoLeader(err) { 106 return true 107 } 108 109 // Reads are safe to retry for stream errors, such as if a server was 110 // being shut down. 111 info, ok := args.(structs.RPCInfo) 112 if ok && info.IsRead() && lib.IsErrEOF(err) { 113 return true 114 } 115 116 return false 117 } 118 119 // RemoteStreamingRpcHandler is used to make a streaming RPC call to a remote 120 // server. 121 func (c *Client) RemoteStreamingRpcHandler(method string) (structs.StreamingRpcHandler, error) { 122 server := c.servers.FindServer() 123 if server == nil { 124 return nil, noServersErr 125 } 126 127 conn, err := c.streamingRpcConn(server, method) 128 if err != nil { 129 // Move off to another server 130 c.rpcLogger.Error("error performing RPC to server", "error", err, "rpc", method, "server", server.Addr) 131 c.servers.NotifyFailedServer(server) 132 return nil, err 133 } 134 135 return bridgedStreamingRpcHandler(conn), nil 136 } 137 138 // bridgedStreamingRpcHandler creates a bridged streaming RPC handler by copying 139 // data between the two sides. 140 func bridgedStreamingRpcHandler(sideA io.ReadWriteCloser) structs.StreamingRpcHandler { 141 return func(sideB io.ReadWriteCloser) { 142 defer sideA.Close() 143 defer sideB.Close() 144 structs.Bridge(sideA, sideB) 145 } 146 } 147 148 // streamingRpcConn is used to retrieve a connection to a server to conduct a 149 // streaming RPC. 150 func (c *Client) streamingRpcConn(server *servers.Server, method string) (net.Conn, error) { 151 // Dial the server 152 conn, err := net.DialTimeout("tcp", server.Addr.String(), 10*time.Second) 153 if err != nil { 154 return nil, err 155 } 156 157 // Cast to TCPConn 158 if tcp, ok := conn.(*net.TCPConn); ok { 159 tcp.SetKeepAlive(true) 160 tcp.SetNoDelay(true) 161 } 162 163 // Check if TLS is enabled 164 c.tlsWrapLock.RLock() 165 tlsWrap := c.tlsWrap 166 c.tlsWrapLock.RUnlock() 167 168 if tlsWrap != nil { 169 // Switch the connection into TLS mode 170 if _, err := conn.Write([]byte{byte(pool.RpcTLS)}); err != nil { 171 conn.Close() 172 return nil, err 173 } 174 175 // Wrap the connection in a TLS client 176 tlsConn, err := tlsWrap(c.Region(), conn) 177 if err != nil { 178 conn.Close() 179 return nil, err 180 } 181 conn = tlsConn 182 } 183 184 // Write the multiplex byte to set the mode 185 if _, err := conn.Write([]byte{byte(pool.RpcStreaming)}); err != nil { 186 conn.Close() 187 return nil, err 188 } 189 190 // Send the header 191 encoder := codec.NewEncoder(conn, structs.MsgpackHandle) 192 decoder := codec.NewDecoder(conn, structs.MsgpackHandle) 193 header := structs.StreamingRpcHeader{ 194 Method: method, 195 } 196 if err := encoder.Encode(header); err != nil { 197 conn.Close() 198 return nil, err 199 } 200 201 // Wait for the acknowledgement 202 var ack structs.StreamingRpcAck 203 if err := decoder.Decode(&ack); err != nil { 204 conn.Close() 205 return nil, err 206 } 207 208 if ack.Error != "" { 209 conn.Close() 210 return nil, errors.New(ack.Error) 211 } 212 213 return conn, nil 214 } 215 216 // setupClientRpc is used to setup the Client's RPC endpoints 217 func (c *Client) setupClientRpc() { 218 // Initialize the RPC handlers 219 c.endpoints.ClientStats = &ClientStats{c} 220 c.endpoints.FileSystem = NewFileSystemEndpoint(c) 221 c.endpoints.Allocations = NewAllocationsEndpoint(c) 222 c.endpoints.Agent = NewAgentEndpoint(c) 223 224 // Create the RPC Server 225 c.rpcServer = rpc.NewServer() 226 227 // Register the endpoints with the RPC server 228 c.setupClientRpcServer(c.rpcServer) 229 230 go c.rpcConnListener() 231 } 232 233 // setupClientRpcServer is used to populate a client RPC server with endpoints. 234 func (c *Client) setupClientRpcServer(server *rpc.Server) { 235 // Register the endpoints 236 server.Register(c.endpoints.ClientStats) 237 server.Register(c.endpoints.FileSystem) 238 server.Register(c.endpoints.Allocations) 239 server.Register(c.endpoints.Agent) 240 } 241 242 // rpcConnListener is a long lived function that listens for new connections 243 // being made on the connection pool and starts an RPC listener for each 244 // connection. 245 func (c *Client) rpcConnListener() { 246 // Make a channel for new connections. 247 conns := make(chan *yamux.Session, 4) 248 c.connPool.SetConnListener(conns) 249 250 for { 251 select { 252 case <-c.shutdownCh: 253 return 254 case session, ok := <-conns: 255 if !ok { 256 continue 257 } 258 259 go c.listenConn(session) 260 } 261 } 262 } 263 264 // listenConn is used to listen for connections being made from the server on 265 // pre-existing connection. This should be called in a goroutine. 266 func (c *Client) listenConn(s *yamux.Session) { 267 for { 268 conn, err := s.Accept() 269 if err != nil { 270 if s.IsClosed() { 271 return 272 } 273 274 c.rpcLogger.Error("failed to accept RPC conn", "error", err) 275 continue 276 } 277 278 go c.handleConn(conn) 279 metrics.IncrCounter([]string{"client", "rpc", "accept_conn"}, 1) 280 } 281 } 282 283 // handleConn is used to determine if this is a RPC or Streaming RPC connection and 284 // invoke the correct handler 285 func (c *Client) handleConn(conn net.Conn) { 286 // Read a single byte 287 buf := make([]byte, 1) 288 if _, err := conn.Read(buf); err != nil { 289 if err != io.EOF { 290 c.rpcLogger.Error("error reading byte", "error", err) 291 } 292 conn.Close() 293 return 294 } 295 296 // Switch on the byte 297 switch pool.RPCType(buf[0]) { 298 case pool.RpcNomad: 299 c.handleNomadConn(conn) 300 301 case pool.RpcStreaming: 302 c.handleStreamingConn(conn) 303 304 default: 305 c.rpcLogger.Error("unrecognized RPC byte", "byte", buf[0]) 306 conn.Close() 307 return 308 } 309 } 310 311 // handleNomadConn is used to handle a single Nomad RPC connection. 312 func (c *Client) handleNomadConn(conn net.Conn) { 313 defer conn.Close() 314 rpcCodec := pool.NewServerCodec(conn) 315 for { 316 select { 317 case <-c.shutdownCh: 318 return 319 default: 320 } 321 322 if err := c.rpcServer.ServeRequest(rpcCodec); err != nil { 323 if err != io.EOF && !strings.Contains(err.Error(), "closed") { 324 c.rpcLogger.Error("error performing RPC", "error", err, "addr", conn.RemoteAddr()) 325 metrics.IncrCounter([]string{"client", "rpc", "request_error"}, 1) 326 } 327 return 328 } 329 metrics.IncrCounter([]string{"client", "rpc", "request"}, 1) 330 } 331 } 332 333 // handleStreamingConn is used to handle a single Streaming Nomad RPC connection. 334 func (c *Client) handleStreamingConn(conn net.Conn) { 335 defer conn.Close() 336 337 // Decode the header 338 var header structs.StreamingRpcHeader 339 decoder := codec.NewDecoder(conn, structs.MsgpackHandle) 340 if err := decoder.Decode(&header); err != nil { 341 if err != io.EOF && !strings.Contains(err.Error(), "closed") { 342 c.rpcLogger.Error("error performing streaming RPC", "error", err, "addr", conn.RemoteAddr()) 343 metrics.IncrCounter([]string{"client", "streaming_rpc", "request_error"}, 1) 344 } 345 346 return 347 } 348 349 ack := structs.StreamingRpcAck{} 350 handler, err := c.streamingRpcs.GetHandler(header.Method) 351 if err != nil { 352 c.rpcLogger.Error("streaming RPC error", "addr", conn.RemoteAddr(), "error", err) 353 metrics.IncrCounter([]string{"client", "streaming_rpc", "request_error"}, 1) 354 ack.Error = err.Error() 355 } 356 357 // Send the acknowledgement 358 encoder := codec.NewEncoder(conn, structs.MsgpackHandle) 359 if err := encoder.Encode(ack); err != nil { 360 conn.Close() 361 return 362 } 363 364 if ack.Error != "" { 365 return 366 } 367 368 // Invoke the handler 369 metrics.IncrCounter([]string{"client", "streaming_rpc", "request"}, 1) 370 handler(conn) 371 } 372 373 // resolveServer given a sever's address as a string, return it's resolved 374 // net.Addr or an error. 375 func resolveServer(s string) (net.Addr, error) { 376 const defaultClientPort = "4647" // default client RPC port 377 host, port, err := net.SplitHostPort(s) 378 if err != nil { 379 if strings.Contains(err.Error(), "missing port") { 380 host = s 381 port = defaultClientPort 382 } else { 383 return nil, err 384 } 385 } 386 return net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) 387 } 388 389 // Ping is used to ping a particular server and returns whether it is healthy or 390 // a potential error. 391 func (c *Client) Ping(srv net.Addr) error { 392 var reply struct{} 393 err := c.connPool.RPC(c.Region(), srv, c.RPCMajorVersion(), "Status.Ping", struct{}{}, &reply) 394 return err 395 } 396 397 // rpcRetryWatcher returns a channel that will be closed if an event happens 398 // such that we expect the next RPC to be successful. 399 func (c *Client) rpcRetryWatcher() <-chan struct{} { 400 c.rpcRetryLock.Lock() 401 defer c.rpcRetryLock.Unlock() 402 403 if c.rpcRetryCh == nil { 404 c.rpcRetryCh = make(chan struct{}) 405 } 406 407 return c.rpcRetryCh 408 } 409 410 // fireRpcRetryWatcher causes any RPC retryloops to retry their RPCs because we 411 // believe the will be successful. 412 func (c *Client) fireRpcRetryWatcher() { 413 c.rpcRetryLock.Lock() 414 defer c.rpcRetryLock.Unlock() 415 if c.rpcRetryCh != nil { 416 close(c.rpcRetryCh) 417 c.rpcRetryCh = nil 418 } 419 }