github.com/emate/nomad@v0.8.2-wo-binpacking/nomad/client_rpc.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "net" 7 "time" 8 9 multierror "github.com/hashicorp/go-multierror" 10 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 11 "github.com/hashicorp/nomad/helper/pool" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/yamux" 14 "github.com/ugorji/go/codec" 15 ) 16 17 // nodeConnState is used to track connection information about a Nomad Client. 18 type nodeConnState struct { 19 // Session holds the multiplexed yamux Session for dialing back. 20 Session *yamux.Session 21 22 // Established is when the connection was established. 23 Established time.Time 24 25 // Ctx is the full RPC context 26 Ctx *RPCContext 27 } 28 29 // getNodeConn returns the connection to the given node and whether it exists. 30 func (s *Server) getNodeConn(nodeID string) (*nodeConnState, bool) { 31 s.nodeConnsLock.RLock() 32 defer s.nodeConnsLock.RUnlock() 33 conns, ok := s.nodeConns[nodeID] 34 35 // Return the latest conn 36 var state *nodeConnState 37 for _, conn := range conns { 38 if state == nil || state.Established.Before(conn.Established) { 39 state = conn 40 } 41 } 42 43 // Shouldn't happen but rather be safe 44 if state == nil { 45 s.logger.Printf("[WARN] nomad.client_rpc: node %q exists in node connection map without any connection", nodeID) 46 return nil, false 47 } 48 49 return state, ok 50 } 51 52 // connectedNodes returns the set of nodes we have a connection with. 53 func (s *Server) connectedNodes() map[string]time.Time { 54 s.nodeConnsLock.RLock() 55 defer s.nodeConnsLock.RUnlock() 56 nodes := make(map[string]time.Time, len(s.nodeConns)) 57 for nodeID, conns := range s.nodeConns { 58 for _, conn := range conns { 59 if nodes[nodeID].Before(conn.Established) { 60 nodes[nodeID] = conn.Established 61 } 62 } 63 } 64 return nodes 65 } 66 67 // addNodeConn adds the mapping between a node and its session. 68 func (s *Server) addNodeConn(ctx *RPCContext) { 69 // Hotpath the no-op 70 if ctx == nil || ctx.NodeID == "" { 71 return 72 } 73 74 s.nodeConnsLock.Lock() 75 defer s.nodeConnsLock.Unlock() 76 77 // Capture the tracked connections so far 78 currentConns := s.nodeConns[ctx.NodeID] 79 80 // Check if we already have the connection. If we do, just update the 81 // establish time. 82 for _, c := range currentConns { 83 if c.Ctx.Conn.LocalAddr().String() == ctx.Conn.LocalAddr().String() && 84 c.Ctx.Conn.RemoteAddr().String() == ctx.Conn.RemoteAddr().String() { 85 c.Established = time.Now() 86 return 87 } 88 } 89 90 // Add the new conn 91 s.nodeConns[ctx.NodeID] = append(s.nodeConns[ctx.NodeID], &nodeConnState{ 92 Session: ctx.Session, 93 Established: time.Now(), 94 Ctx: ctx, 95 }) 96 } 97 98 // removeNodeConn removes the mapping between a node and its session. 99 func (s *Server) removeNodeConn(ctx *RPCContext) { 100 // Hotpath the no-op 101 if ctx == nil || ctx.NodeID == "" { 102 return 103 } 104 105 s.nodeConnsLock.Lock() 106 defer s.nodeConnsLock.Unlock() 107 conns, ok := s.nodeConns[ctx.NodeID] 108 if !ok { 109 return 110 } 111 112 // It is important that we check that the connection being removed is the 113 // actual stored connection for the client. It is possible for the client to 114 // dial various addresses that all route to the same server. The most common 115 // case for this is the original address the client uses to connect to the 116 // server differs from the advertised address sent by the heartbeat. 117 for i, conn := range conns { 118 if conn.Ctx.Conn.LocalAddr().String() == ctx.Conn.LocalAddr().String() && 119 conn.Ctx.Conn.RemoteAddr().String() == ctx.Conn.RemoteAddr().String() { 120 121 if len(conns) == 1 { 122 // We are deleting the last conn, remove it from the map 123 delete(s.nodeConns, ctx.NodeID) 124 } else { 125 // Slice out the connection we are deleting 126 s.nodeConns[ctx.NodeID] = append(s.nodeConns[ctx.NodeID][:i], s.nodeConns[ctx.NodeID][i+1:]...) 127 } 128 129 return 130 } 131 } 132 } 133 134 // serverWithNodeConn is used to determine which remote server has the most 135 // recent connection to the given node. The local server is not queried. 136 // ErrNoNodeConn is returned if all local peers could be queried but did not 137 // have a connection to the node. Otherwise if a connection could not be found 138 // and there were RPC errors, an error is returned. 139 func (s *Server) serverWithNodeConn(nodeID, region string) (*serverParts, error) { 140 // We skip ourselves. 141 selfAddr := s.LocalMember().Addr.String() 142 143 // Build the request 144 req := &structs.NodeSpecificRequest{ 145 NodeID: nodeID, 146 QueryOptions: structs.QueryOptions{ 147 Region: s.config.Region, 148 }, 149 } 150 151 // Select the list of servers to check based on what region we are querying 152 s.peerLock.RLock() 153 154 var rawTargets []*serverParts 155 if region == s.Region() { 156 rawTargets = make([]*serverParts, 0, len(s.localPeers)) 157 for _, srv := range s.localPeers { 158 rawTargets = append(rawTargets, srv) 159 } 160 } else { 161 peers, ok := s.peers[region] 162 if !ok { 163 s.peerLock.RUnlock() 164 return nil, structs.ErrNoRegionPath 165 } 166 rawTargets = peers 167 } 168 169 targets := make([]*serverParts, 0, len(rawTargets)) 170 for _, target := range rawTargets { 171 targets = append(targets, target.Copy()) 172 } 173 s.peerLock.RUnlock() 174 175 // connections is used to store the servers that have connections to the 176 // requested node. 177 var mostRecentServer *serverParts 178 var mostRecent time.Time 179 180 var rpcErr multierror.Error 181 for _, server := range targets { 182 if server.Addr.String() == selfAddr { 183 continue 184 } 185 186 // Make the RPC 187 var resp structs.NodeConnQueryResponse 188 err := s.connPool.RPC(s.config.Region, server.Addr, server.MajorVersion, 189 "Status.HasNodeConn", &req, &resp) 190 if err != nil { 191 multierror.Append(&rpcErr, fmt.Errorf("failed querying server %q: %v", server.Addr.String(), err)) 192 continue 193 } 194 195 if resp.Connected && resp.Established.After(mostRecent) { 196 mostRecentServer = server 197 mostRecent = resp.Established 198 } 199 } 200 201 // Return an error if there is no route to the node. 202 if mostRecentServer == nil { 203 if err := rpcErr.ErrorOrNil(); err != nil { 204 return nil, err 205 } 206 207 return nil, structs.ErrNoNodeConn 208 } 209 210 return mostRecentServer, nil 211 } 212 213 // NodeRpc is used to make an RPC call to a node. The method takes the 214 // Yamux session for the node and the method to be called. 215 func NodeRpc(session *yamux.Session, method string, args, reply interface{}) error { 216 // Open a new session 217 stream, err := session.Open() 218 if err != nil { 219 return err 220 } 221 defer stream.Close() 222 223 // Write the RpcNomad byte to set the mode 224 if _, err := stream.Write([]byte{byte(pool.RpcNomad)}); err != nil { 225 stream.Close() 226 return err 227 } 228 229 // Make the RPC 230 err = msgpackrpc.CallWithCodec(pool.NewClientCodec(stream), method, args, reply) 231 if err != nil { 232 return err 233 } 234 235 return nil 236 } 237 238 // NodeStreamingRpc is used to make a streaming RPC call to a node. The method 239 // takes the Yamux session for the node and the method to be called. It conducts 240 // the initial handshake and returns a connection to be used or an error. It is 241 // the callers responsibility to close the connection if there is no error. 242 func NodeStreamingRpc(session *yamux.Session, method string) (net.Conn, error) { 243 // Open a new session 244 stream, err := session.Open() 245 if err != nil { 246 return nil, err 247 } 248 249 // Write the RpcNomad byte to set the mode 250 if _, err := stream.Write([]byte{byte(pool.RpcStreaming)}); err != nil { 251 stream.Close() 252 return nil, err 253 } 254 255 // Send the header 256 encoder := codec.NewEncoder(stream, structs.MsgpackHandle) 257 decoder := codec.NewDecoder(stream, structs.MsgpackHandle) 258 header := structs.StreamingRpcHeader{ 259 Method: method, 260 } 261 if err := encoder.Encode(header); err != nil { 262 stream.Close() 263 return nil, err 264 } 265 266 // Wait for the acknowledgement 267 var ack structs.StreamingRpcAck 268 if err := decoder.Decode(&ack); err != nil { 269 stream.Close() 270 return nil, err 271 } 272 273 if ack.Error != "" { 274 stream.Close() 275 return nil, errors.New(ack.Error) 276 } 277 278 return stream, nil 279 } 280 281 // findNodeConnAndForward is a helper for finding the server with a connection 282 // to the given node and forwarding the RPC to the correct server. This does not 283 // work for streaming RPCs. 284 func findNodeConnAndForward(srv *Server, nodeID, method string, args, reply interface{}) error { 285 // Determine the Server that has a connection to the node. 286 srvWithConn, err := srv.serverWithNodeConn(nodeID, srv.Region()) 287 if err != nil { 288 return err 289 } 290 291 if srvWithConn == nil { 292 return structs.ErrNoNodeConn 293 } 294 295 return srv.forwardServer(srvWithConn, method, args, reply) 296 }