github.com/dbernstein1/tyk@v2.9.0-beta9-dl-apic+incompatible/rpc/rpc_client.go (about) 1 package rpc 2 3 import ( 4 "crypto/tls" 5 "errors" 6 "net" 7 "strconv" 8 "strings" 9 "sync" 10 "sync/atomic" 11 "time" 12 13 "github.com/sirupsen/logrus" 14 15 "github.com/gocraft/health" 16 uuid "github.com/satori/go.uuid" 17 18 "github.com/TykTechnologies/gorpc" 19 ) 20 21 var ( 22 GlobalRPCCallTimeout = 30 * time.Second 23 GlobalRPCPingTimeout = 60 * time.Second 24 Log = &logrus.Logger{} 25 Instrument *health.Stream 26 27 clientSingleton *gorpc.Client 28 clientSingletonMu sync.Mutex 29 funcClientSingleton *gorpc.DispatcherClient 30 clientIsConnected bool 31 32 dispatcher = gorpc.NewDispatcher() 33 addedFuncs = make(map[string]bool) 34 35 config Config 36 getGroupLoginCallback func(string, string) interface{} 37 emergencyModeCallback func() 38 emergencyModeLoadedCallback func() 39 40 // rpcLoadCount is a counter to check if this is a cold boot 41 rpcLoadCount int 42 rpcEmergencyMode bool 43 rpcEmergencyModeLoaded bool 44 45 killChan = make(chan int) 46 killed bool 47 id string 48 49 rpcLoginMu sync.Mutex 50 reLoginRunning uint32 51 52 rpcConnectMu sync.Mutex 53 ) 54 55 const ( 56 ClientSingletonCall = "gorpcClientCall" 57 FuncClientSingletonCall = "gorpcDispatcherClientCall" 58 ) 59 60 type Config struct { 61 UseSSL bool `json:"use_ssl"` 62 SSLInsecureSkipVerify bool `json:"ssl_insecure_skip_verify"` 63 ConnectionString string `json:"connection_string"` 64 RPCKey string `json:"rpc_key"` 65 APIKey string `json:"api_key"` 66 GroupID string `json:"group_id"` 67 CallTimeout int `json:"call_timeout"` 68 PingTimeout int `json:"ping_timeout"` 69 RPCPoolSize int `json:"rpc_pool_size"` 70 } 71 72 func IsEmergencyMode() bool { 73 return rpcEmergencyMode 74 } 75 76 func LoadCount() int { 77 return rpcLoadCount 78 } 79 80 func Reset() { 81 clientSingleton.Stop() 82 clientIsConnected = false 83 clientSingleton = nil 84 funcClientSingleton = nil 85 rpcLoadCount = 0 86 rpcEmergencyMode = false 87 rpcEmergencyModeLoaded = false 88 } 89 90 func ResetEmergencyMode() { 91 rpcEmergencyModeLoaded = false 92 rpcEmergencyMode = false 93 } 94 95 func EmitErrorEvent(jobName string, funcName string, err error) { 96 if Instrument == nil { 97 return 98 } 99 100 job := Instrument.NewJob(jobName) 101 if emitErr := job.EventErr(funcName, err); emitErr != nil { 102 Log.WithError(emitErr).WithFields(logrus.Fields{ 103 "jobName": jobName, 104 "funcName": funcName, 105 }) 106 } 107 } 108 109 func EmitErrorEventKv(jobName string, funcName string, err error, kv map[string]string) { 110 if Instrument == nil { 111 return 112 } 113 114 job := Instrument.NewJob(jobName) 115 if emitErr := job.EventErrKv(funcName, err, kv); emitErr != nil { 116 Log.WithError(emitErr).WithFields(logrus.Fields{ 117 "jobName": jobName, 118 "funcName": funcName, 119 "kv": kv, 120 }) 121 } 122 } 123 124 // Connect will establish a connection to the RPC server specified in connection options 125 func Connect(connConfig Config, suppressRegister bool, dispatcherFuncs map[string]interface{}, 126 getGroupLoginFunc func(string, string) interface{}, 127 emergencyModeFunc func(), 128 emergencyModeLoadedFunc func()) bool { 129 rpcConnectMu.Lock() 130 defer rpcConnectMu.Unlock() 131 132 config = connConfig 133 getGroupLoginCallback = getGroupLoginFunc 134 emergencyModeCallback = emergencyModeFunc 135 emergencyModeLoadedCallback = emergencyModeLoadedFunc 136 137 if clientIsConnected { 138 Log.Debug("Using RPC singleton for connection") 139 return true 140 } 141 142 if clientSingleton != nil { 143 return rpcEmergencyMode != true 144 } 145 146 // RPC Client is unset 147 // Set up the cache 148 Log.Info("Setting new RPC connection!") 149 150 connID := uuid.NewV4().String() 151 152 // Length should fit into 1 byte. Protection if we decide change uuid in future. 153 if len(connID) > 255 { 154 panic("connID is too long") 155 } 156 157 if config.UseSSL { 158 clientCfg := &tls.Config{ 159 InsecureSkipVerify: config.SSLInsecureSkipVerify, 160 } 161 162 clientSingleton = gorpc.NewTLSClient(config.ConnectionString, clientCfg) 163 } else { 164 clientSingleton = gorpc.NewTCPClient(config.ConnectionString) 165 } 166 167 if Log.Level != logrus.DebugLevel { 168 clientSingleton.LogError = gorpc.NilErrorLogger 169 } 170 171 clientSingleton.OnConnect = onConnectFunc 172 173 clientSingleton.Conns = config.RPCPoolSize 174 if clientSingleton.Conns == 0 { 175 clientSingleton.Conns = 20 176 } 177 178 clientSingleton.Dial = func(addr string) (conn net.Conn, err error) { 179 dialer := &net.Dialer{ 180 Timeout: 10 * time.Second, 181 KeepAlive: 30 * time.Second, 182 } 183 184 useSSL := config.UseSSL 185 186 if useSSL { 187 cfg := &tls.Config{ 188 InsecureSkipVerify: config.SSLInsecureSkipVerify, 189 } 190 191 conn, err = tls.DialWithDialer(dialer, "tcp", addr, cfg) 192 } else { 193 conn, err = dialer.Dial("tcp", addr) 194 } 195 196 if err != nil { 197 EmitErrorEventKv( 198 ClientSingletonCall, 199 "dial", 200 err, 201 map[string]string{ 202 "addr": addr, 203 "useSSL": strconv.FormatBool(useSSL), 204 }, 205 ) 206 return 207 } 208 209 conn.Write([]byte("proto2")) 210 conn.Write([]byte{byte(len(connID))}) 211 conn.Write([]byte(connID)) 212 return conn, nil 213 } 214 clientSingleton.Start() 215 216 loadDispatcher(dispatcherFuncs) 217 218 if funcClientSingleton == nil { 219 funcClientSingleton = dispatcher.NewFuncClient(clientSingleton) 220 } 221 222 if !Login() { 223 return false 224 } 225 226 if !suppressRegister { 227 register() 228 go checkDisconnect() 229 } 230 231 return true 232 } 233 234 func reAttemptLogin(err error) bool { 235 if atomic.LoadUint32(&reLoginRunning) == 1 { 236 return false 237 } 238 atomic.StoreUint32(&reLoginRunning, 1) 239 240 rpcLoginMu.Lock() 241 if rpcLoadCount == 0 && !rpcEmergencyModeLoaded { 242 Log.Warning("[RPC Store] --> Detected cold start, attempting to load from cache") 243 Log.Warning("[RPC Store] ----> Found APIs... beginning emergency load") 244 rpcEmergencyModeLoaded = true 245 if emergencyModeLoadedCallback != nil { 246 go emergencyModeLoadedCallback() 247 } 248 } 249 rpcLoginMu.Unlock() 250 251 time.Sleep(time.Second * 3) 252 atomic.StoreUint32(&reLoginRunning, 0) 253 254 if strings.Contains(err.Error(), "Cannot obtain response during timeout") { 255 reConnect() 256 return false 257 } 258 259 Log.Warning("[RPC Store] Login failed, waiting 3s to re-attempt") 260 261 return Login() 262 } 263 264 func GroupLogin() bool { 265 if getGroupLoginCallback == nil { 266 Log.Error("GroupLogin call back is not set") 267 return false 268 } 269 270 groupLoginData := getGroupLoginCallback(config.APIKey, config.GroupID) 271 ok, err := FuncClientSingleton("LoginWithGroup", groupLoginData) 272 if err != nil { 273 Log.WithError(err).Error("RPC Login failed") 274 EmitErrorEventKv( 275 FuncClientSingletonCall, 276 "LoginWithGroup", 277 err, 278 map[string]string{ 279 "GroupID": config.GroupID, 280 }, 281 ) 282 rpcEmergencyMode = true 283 go reAttemptLogin(err) 284 return false 285 } 286 287 if ok == false { 288 Log.Error("RPC Login incorrect") 289 rpcEmergencyMode = true 290 go reAttemptLogin(errors.New("Login incorrect")) 291 return false 292 } 293 Log.Debug("[RPC Store] Group Login complete") 294 rpcLoadCount++ 295 296 // Recovery 297 if rpcEmergencyMode { 298 rpcEmergencyMode = false 299 rpcEmergencyModeLoaded = false 300 if emergencyModeCallback != nil { 301 emergencyModeCallback() 302 } 303 } 304 305 return true 306 } 307 308 func Login() bool { 309 Log.Debug("[RPC Store] Login initiated") 310 311 if len(config.APIKey) == 0 { 312 Log.Fatal("No API Key set!") 313 } 314 315 // If we have a group ID, lets login as a group 316 if config.GroupID != "" { 317 return GroupLogin() 318 } 319 320 ok, err := FuncClientSingleton("Login", config.APIKey) 321 if err != nil { 322 Log.WithError(err).Error("RPC Login failed") 323 EmitErrorEvent(FuncClientSingletonCall, "Login", err) 324 rpcEmergencyMode = true 325 go reAttemptLogin(err) 326 return false 327 } 328 329 if ok == false { 330 Log.Error("RPC Login incorrect") 331 rpcEmergencyMode = true 332 go reAttemptLogin(errors.New("Login incorrect")) 333 return false 334 } 335 Log.Debug("[RPC Store] Login complete") 336 rpcLoadCount++ 337 338 if rpcEmergencyMode { 339 rpcEmergencyMode = false 340 rpcEmergencyModeLoaded = false 341 if emergencyModeCallback != nil { 342 emergencyModeCallback() 343 } 344 } 345 346 return true 347 } 348 349 func FuncClientSingleton(funcName string, request interface{}) (interface{}, error) { 350 return funcClientSingleton.CallTimeout(funcName, request, GlobalRPCCallTimeout) 351 } 352 353 func onConnectFunc(conn net.Conn) (net.Conn, string, error) { 354 clientSingletonMu.Lock() 355 defer clientSingletonMu.Unlock() 356 357 clientIsConnected = true 358 remoteAddr := conn.RemoteAddr().String() 359 Log.WithField("remoteAddr", remoteAddr).Debug("connected to RPC server") 360 361 return conn, remoteAddr, nil 362 } 363 364 func Disconnect() bool { 365 clientIsConnected = false 366 return true 367 } 368 369 func reConnect() { 370 // no-op, let the gorpc client handle it. 371 } 372 373 func register() { 374 id = uuid.NewV4().String() 375 Log.Debug("RPC Client registered") 376 } 377 378 func checkDisconnect() { 379 res := <-killChan 380 Log.WithField("res", res).Info("RPC Client disconnecting") 381 killed = true 382 Disconnect() 383 } 384 385 func loadDispatcher(dispatcherFuncs map[string]interface{}) { 386 for funcName, funcBody := range dispatcherFuncs { 387 if addedFuncs[funcName] { 388 continue 389 } 390 dispatcher.AddFunc(funcName, funcBody) 391 addedFuncs[funcName] = true 392 } 393 }