github.com/Tyktechnologies/tyk@v2.9.5+incompatible/gateway/rpc_storage_handler.go (about)

     1  package gateway
     2  
     3  import (
     4  	"strconv"
     5  	"strings"
     6  	"time"
     7  
     8  	cache "github.com/pmylund/go-cache"
     9  
    10  	"github.com/TykTechnologies/tyk/rpc"
    11  
    12  	"github.com/TykTechnologies/tyk/apidef"
    13  	"github.com/TykTechnologies/tyk/config"
    14  	"github.com/TykTechnologies/tyk/storage"
    15  	"github.com/go-redis/redis"
    16  
    17  	"github.com/sirupsen/logrus"
    18  )
    19  
    20  var (
    21  	dispatcherFuncs = map[string]interface{}{
    22  		"Login": func(clientAddr, userKey string) bool {
    23  			return false
    24  		},
    25  		"LoginWithGroup": func(clientAddr string, groupData *apidef.GroupLoginRequest) bool {
    26  			return false
    27  		},
    28  		"GetKey": func(keyName string) (string, error) {
    29  			return "", nil
    30  		},
    31  		"SetKey": func(ibd *apidef.InboundData) error {
    32  			return nil
    33  		},
    34  		"GetExp": func(keyName string) (int64, error) {
    35  			return 0, nil
    36  		},
    37  		"GetKeys": func(keyName string) ([]string, error) {
    38  			return nil, nil
    39  		},
    40  		"DeleteKey": func(keyName string) (bool, error) {
    41  			return true, nil
    42  		},
    43  		"DeleteRawKey": func(keyName string) (bool, error) {
    44  			return true, nil
    45  		},
    46  		"GetKeysAndValues": func(searchString string) (*apidef.KeysValuesPair, error) {
    47  			return nil, nil
    48  		},
    49  		"GetKeysAndValuesWithFilter": func(searchString string) (*apidef.KeysValuesPair, error) {
    50  			return nil, nil
    51  		},
    52  		"DeleteKeys": func(keys []string) (bool, error) {
    53  			return true, nil
    54  		},
    55  		"Decrement": func(keyName string) error {
    56  			return nil
    57  		},
    58  		"IncrememntWithExpire": func(ibd *apidef.InboundData) (int64, error) {
    59  			return 0, nil
    60  		},
    61  		"AppendToSet": func(ibd *apidef.InboundData) error {
    62  			return nil
    63  		},
    64  		"SetRollingWindow": func(ibd *apidef.InboundData) (int, error) {
    65  			return 0, nil
    66  		},
    67  		"GetApiDefinitions": func(dr *apidef.DefRequest) (string, error) {
    68  			return "", nil
    69  		},
    70  		"GetPolicies": func(orgId string) (string, error) {
    71  			return "", nil
    72  		},
    73  		"PurgeAnalyticsData": func(data string) error {
    74  			return nil
    75  		},
    76  		"CheckReload": func(clientAddr, orgId string) (bool, error) {
    77  			return false, nil
    78  		},
    79  		"GetKeySpaceUpdate": func(clientAddr, orgId string) ([]string, error) {
    80  			return nil, nil
    81  		},
    82  		"GetGroupKeySpaceUpdate": func(clientAddr string, groupData *apidef.GroupKeySpaceRequest) ([]string, error) {
    83  			return nil, nil
    84  		},
    85  		"Ping": func() bool {
    86  			return false
    87  		},
    88  	}
    89  )
    90  
    91  // RPCStorageHandler is a storage manager that uses the redis database.
    92  type RPCStorageHandler struct {
    93  	KeyPrefix        string
    94  	HashKeys         bool
    95  	SuppressRegister bool
    96  }
    97  
    98  var RPCGlobalCache = cache.New(30*time.Second, 15*time.Second)
    99  
   100  // Connect will establish a connection to the RPC
   101  func (r *RPCStorageHandler) Connect() bool {
   102  	slaveOptions := config.Global().SlaveOptions
   103  	rpcConfig := rpc.Config{
   104  		UseSSL:                slaveOptions.UseSSL,
   105  		SSLInsecureSkipVerify: slaveOptions.SSLInsecureSkipVerify,
   106  		ConnectionString:      slaveOptions.ConnectionString,
   107  		RPCKey:                slaveOptions.RPCKey,
   108  		APIKey:                slaveOptions.APIKey,
   109  		GroupID:               slaveOptions.GroupID,
   110  		CallTimeout:           slaveOptions.CallTimeout,
   111  		PingTimeout:           slaveOptions.PingTimeout,
   112  		RPCPoolSize:           slaveOptions.RPCPoolSize,
   113  	}
   114  
   115  	return rpc.Connect(
   116  		rpcConfig,
   117  		r.SuppressRegister,
   118  		dispatcherFuncs,
   119  		func(userKey string, groupID string) interface{} {
   120  			return apidef.GroupLoginRequest{
   121  				UserKey: userKey,
   122  				GroupID: groupID,
   123  			}
   124  		},
   125  		func() {
   126  			reloadURLStructure(nil)
   127  		},
   128  		DoReload,
   129  	)
   130  }
   131  
   132  func (r *RPCStorageHandler) hashKey(in string) string {
   133  	if !r.HashKeys {
   134  		// Not hashing? Return the raw key
   135  		return in
   136  	}
   137  	return storage.HashStr(in)
   138  }
   139  
   140  func (r *RPCStorageHandler) fixKey(keyName string) string {
   141  	setKeyName := r.KeyPrefix + r.hashKey(keyName)
   142  
   143  	log.Debug("Input key was: ", setKeyName)
   144  
   145  	return setKeyName
   146  }
   147  
   148  func (r *RPCStorageHandler) cleanKey(keyName string) string {
   149  	setKeyName := strings.Replace(keyName, r.KeyPrefix, "", 1)
   150  	return setKeyName
   151  }
   152  
   153  // GetKey will retrieve a key from the database
   154  func (r *RPCStorageHandler) GetKey(keyName string) (string, error) {
   155  	start := time.Now() // get current time
   156  	//	log.Debug("[STORE] Getting WAS: ", keyName)
   157  	//  log.Debug("[STORE] Getting: ", r.fixKey(keyName))
   158  	value, err := r.GetRawKey(r.fixKey(keyName))
   159  
   160  	elapsed := time.Since(start)
   161  	log.Debug("GetKey took ", elapsed)
   162  
   163  	return value, err
   164  }
   165  
   166  func (r *RPCStorageHandler) GetRawKey(keyName string) (string, error) {
   167  	// Check the cache first
   168  	if config.Global().SlaveOptions.EnableRPCCache {
   169  		log.Debug("Using cache for: ", keyName)
   170  		cachedVal, found := RPCGlobalCache.Get(keyName)
   171  		log.Debug("--> Found? ", found)
   172  		if found {
   173  			return cachedVal.(string), nil
   174  		}
   175  	}
   176  
   177  	value, err := rpc.FuncClientSingleton("GetKey", keyName)
   178  	if err != nil {
   179  		rpc.EmitErrorEventKv(
   180  			rpc.FuncClientSingletonCall,
   181  			"GetKey",
   182  			err,
   183  			map[string]string{
   184  				"keyName": keyName,
   185  			},
   186  		)
   187  		if r.IsAccessError(err) {
   188  			if rpc.Login() {
   189  				return r.GetRawKey(keyName)
   190  			}
   191  		}
   192  		log.Debug("Error trying to get value:", err)
   193  		return "", storage.ErrKeyNotFound
   194  	}
   195  	if config.Global().SlaveOptions.EnableRPCCache {
   196  		// Cache key
   197  		RPCGlobalCache.Set(keyName, value, cache.DefaultExpiration)
   198  	}
   199  	//return hash key without prefix so it doesnt get double prefixed in redis
   200  	return value.(string), nil
   201  }
   202  
   203  func (r *RPCStorageHandler) GetMultiKey(keyNames []string) ([]string, error) {
   204  	var err error
   205  	var value string
   206  
   207  	for _, key := range keyNames {
   208  		value, err = r.GetKey(key)
   209  		if err == nil {
   210  			return []string{value}, nil
   211  		}
   212  	}
   213  
   214  	return nil, err
   215  }
   216  
   217  func (r *RPCStorageHandler) GetExp(keyName string) (int64, error) {
   218  	log.Debug("GetExp called")
   219  	value, err := rpc.FuncClientSingleton("GetExp", r.fixKey(keyName))
   220  	if err != nil {
   221  		rpc.EmitErrorEventKv(
   222  			rpc.FuncClientSingletonCall,
   223  			"GetExp",
   224  			err,
   225  			map[string]string{
   226  				"keyName":      keyName,
   227  				"fixedKeyName": r.fixKey(keyName),
   228  			},
   229  		)
   230  		if r.IsAccessError(err) {
   231  			if rpc.Login() {
   232  				return r.GetExp(keyName)
   233  			}
   234  		}
   235  		log.Error("Error trying to get TTL: ", err)
   236  		return 0, storage.ErrKeyNotFound
   237  	}
   238  	return value.(int64), nil
   239  }
   240  
   241  func (r *RPCStorageHandler) SetExp(keyName string, timeout int64) error {
   242  	log.Error("RPCStorageHandler.SetExp - Not Implemented")
   243  	return nil
   244  }
   245  
   246  // SetKey will create (or update) a key value in the store
   247  func (r *RPCStorageHandler) SetKey(keyName, session string, timeout int64) error {
   248  	start := time.Now() // get current time
   249  	ibd := apidef.InboundData{
   250  		KeyName:      r.fixKey(keyName),
   251  		SessionState: session,
   252  		Timeout:      timeout,
   253  	}
   254  
   255  	_, err := rpc.FuncClientSingleton("SetKey", ibd)
   256  	if err != nil {
   257  		rpc.EmitErrorEventKv(
   258  			rpc.FuncClientSingletonCall,
   259  			"SetKey",
   260  			err,
   261  			map[string]string{
   262  				"keyName":      keyName,
   263  				"fixedKeyName": ibd.KeyName,
   264  			},
   265  		)
   266  
   267  		if r.IsAccessError(err) {
   268  			if rpc.Login() {
   269  				return r.SetKey(keyName, session, timeout)
   270  			}
   271  		}
   272  
   273  		log.Debug("Error trying to set value:", err)
   274  		return err
   275  	}
   276  
   277  	elapsed := time.Since(start)
   278  	log.Debug("SetKey took ", elapsed)
   279  	return nil
   280  
   281  }
   282  
   283  func (r *RPCStorageHandler) SetRawKey(keyName, session string, timeout int64) error {
   284  	return nil
   285  }
   286  
   287  // Decrement will decrement a key in redis
   288  func (r *RPCStorageHandler) Decrement(keyName string) {
   289  	log.Warning("Decrement called")
   290  	_, err := rpc.FuncClientSingleton("Decrement", keyName)
   291  	if err != nil {
   292  		rpc.EmitErrorEventKv(
   293  			rpc.FuncClientSingletonCall,
   294  			"Decrement",
   295  			err,
   296  			map[string]string{
   297  				"keyName": keyName,
   298  			},
   299  		)
   300  	}
   301  	if r.IsAccessError(err) {
   302  		if rpc.Login() {
   303  			r.Decrement(keyName)
   304  			return
   305  		}
   306  	}
   307  }
   308  
   309  // IncrementWithExpire will increment a key in redis
   310  func (r *RPCStorageHandler) IncrememntWithExpire(keyName string, expire int64) int64 {
   311  
   312  	ibd := apidef.InboundData{
   313  		KeyName: keyName,
   314  		Expire:  expire,
   315  	}
   316  
   317  	val, err := rpc.FuncClientSingleton("IncrememntWithExpire", ibd)
   318  	if err != nil {
   319  		rpc.EmitErrorEventKv(
   320  			rpc.FuncClientSingletonCall,
   321  			"IncrememntWithExpire",
   322  			err,
   323  			map[string]string{
   324  				"keyName": keyName,
   325  			},
   326  		)
   327  	}
   328  	if r.IsAccessError(err) {
   329  		if rpc.Login() {
   330  			return r.IncrememntWithExpire(keyName, expire)
   331  		}
   332  	}
   333  
   334  	if val == nil {
   335  		log.Warning("RPC increment returned nil value, returning 0")
   336  		return 0
   337  	}
   338  
   339  	return val.(int64)
   340  
   341  }
   342  
   343  // GetKeys will return all keys according to the filter (filter is a prefix - e.g. tyk.keys.*)
   344  func (r *RPCStorageHandler) GetKeys(filter string) []string {
   345  	log.Error("RPCStorageHandler.GetKeys - Not Implemented")
   346  	return nil
   347  }
   348  
   349  // GetKeysAndValuesWithFilter will return all keys and their values with a filter
   350  func (r *RPCStorageHandler) GetKeysAndValuesWithFilter(filter string) map[string]string {
   351  
   352  	searchStr := r.KeyPrefix + r.hashKey(filter) + "*"
   353  	log.Debug("[STORE] Getting list by: ", searchStr)
   354  
   355  	kvPair, err := rpc.FuncClientSingleton("GetKeysAndValuesWithFilter", searchStr)
   356  	if err != nil {
   357  		rpc.EmitErrorEventKv(
   358  			rpc.FuncClientSingletonCall,
   359  			"GetKeysAndValuesWithFilter",
   360  			err,
   361  			map[string]string{
   362  				"searchStr": searchStr,
   363  			},
   364  		)
   365  
   366  		if r.IsAccessError(err) {
   367  			if rpc.Login() {
   368  				return r.GetKeysAndValuesWithFilter(filter)
   369  			}
   370  		}
   371  
   372  		return nil
   373  	}
   374  
   375  	returnValues := make(map[string]string)
   376  
   377  	for i, v := range kvPair.(*apidef.KeysValuesPair).Keys {
   378  		returnValues[r.cleanKey(v)] = kvPair.(*apidef.KeysValuesPair).Values[i]
   379  	}
   380  
   381  	return returnValues
   382  }
   383  
   384  // GetKeysAndValues will return all keys and their values - not to be used lightly
   385  func (r *RPCStorageHandler) GetKeysAndValues() map[string]string {
   386  
   387  	searchStr := r.KeyPrefix + "*"
   388  
   389  	kvPair, err := rpc.FuncClientSingleton("GetKeysAndValues", searchStr)
   390  	if err != nil {
   391  		rpc.EmitErrorEvent(rpc.FuncClientSingletonCall, "GetKeysAndValues", err)
   392  
   393  		if r.IsAccessError(err) {
   394  			if rpc.Login() {
   395  				return r.GetKeysAndValues()
   396  			}
   397  		}
   398  
   399  		return nil
   400  	}
   401  
   402  	returnValues := make(map[string]string)
   403  	for i, v := range kvPair.(*apidef.KeysValuesPair).Keys {
   404  		returnValues[r.cleanKey(v)] = kvPair.(*apidef.KeysValuesPair).Values[i]
   405  	}
   406  
   407  	return returnValues
   408  
   409  }
   410  
   411  // DeleteKey will remove a key from the database
   412  func (r *RPCStorageHandler) DeleteKey(keyName string) bool {
   413  
   414  	log.Debug("DEL Key was: ", keyName)
   415  	log.Debug("DEL Key became: ", r.fixKey(keyName))
   416  	ok, err := rpc.FuncClientSingleton("DeleteKey", r.fixKey(keyName))
   417  	if err != nil {
   418  		rpc.EmitErrorEventKv(
   419  			rpc.FuncClientSingletonCall,
   420  			"DeleteKey",
   421  			err,
   422  			map[string]string{
   423  				"keyName":      keyName,
   424  				"fixedKeyName": r.fixKey(keyName),
   425  			},
   426  		)
   427  
   428  		if r.IsAccessError(err) {
   429  			if rpc.Login() {
   430  				return r.DeleteKey(keyName)
   431  			}
   432  		}
   433  	}
   434  
   435  	return ok == true
   436  }
   437  
   438  func (r *RPCStorageHandler) DeleteAllKeys() bool {
   439  	log.Warning("Not implementated")
   440  	return false
   441  }
   442  
   443  // DeleteKey will remove a key from the database without prefixing, assumes user knows what they are doing
   444  func (r *RPCStorageHandler) DeleteRawKey(keyName string) bool {
   445  	ok, err := rpc.FuncClientSingleton("DeleteRawKey", keyName)
   446  	if err != nil {
   447  		rpc.EmitErrorEventKv(
   448  			rpc.FuncClientSingletonCall,
   449  			"DeleteRawKey",
   450  			err,
   451  			map[string]string{
   452  				"keyName": keyName,
   453  			},
   454  		)
   455  
   456  		if r.IsAccessError(err) {
   457  			if rpc.Login() {
   458  				return r.DeleteRawKey(keyName)
   459  			}
   460  		}
   461  	}
   462  
   463  	return ok == true
   464  }
   465  
   466  // DeleteKeys will remove a group of keys in bulk
   467  func (r *RPCStorageHandler) DeleteKeys(keys []string) bool {
   468  	if len(keys) > 0 {
   469  		asInterface := make([]string, len(keys))
   470  		for i, v := range keys {
   471  			asInterface[i] = r.fixKey(v)
   472  		}
   473  
   474  		log.Debug("Deleting: ", asInterface)
   475  		ok, err := rpc.FuncClientSingleton("DeleteKeys", asInterface)
   476  		if err != nil {
   477  			rpc.EmitErrorEventKv(
   478  				rpc.FuncClientSingletonCall,
   479  				"DeleteKeys",
   480  				err,
   481  				map[string]string{
   482  					"keys":        strings.Join(keys, ","),
   483  					"asInterface": strings.Join(asInterface, ","),
   484  				},
   485  			)
   486  
   487  			if r.IsAccessError(err) {
   488  				if rpc.Login() {
   489  					return r.DeleteKeys(keys)
   490  				}
   491  			}
   492  		}
   493  
   494  		return ok == true
   495  	}
   496  	log.Debug("RPCStorageHandler called DEL - Nothing to delete")
   497  	return true
   498  }
   499  
   500  // StartPubSubHandler will listen for a signal and run the callback with the message
   501  func (r *RPCStorageHandler) StartPubSubHandler(channel string, callback func(*redis.Message)) error {
   502  	log.Warning("RPCStorageHandler.StartPubSubHandler - NO PUBSUB DEFINED")
   503  	return nil
   504  }
   505  
   506  func (r *RPCStorageHandler) Publish(channel, message string) error {
   507  	log.Warning("RPCStorageHandler.Publish - NO PUBSUB DEFINED")
   508  	return nil
   509  }
   510  
   511  func (r *RPCStorageHandler) GetAndDeleteSet(keyName string) []interface{} {
   512  	log.Error("RPCStorageHandler.GetAndDeleteSet - Not implemented, please disable your purger")
   513  	return nil
   514  }
   515  
   516  func (r *RPCStorageHandler) AppendToSet(keyName, value string) {
   517  	ibd := apidef.InboundData{
   518  		KeyName: keyName,
   519  		Value:   value,
   520  	}
   521  
   522  	_, err := rpc.FuncClientSingleton("AppendToSet", ibd)
   523  	if err != nil {
   524  		rpc.EmitErrorEventKv(
   525  			rpc.FuncClientSingletonCall,
   526  			"AppendToSet",
   527  			err,
   528  			map[string]string{
   529  				"keyName": keyName,
   530  			},
   531  		)
   532  	}
   533  	if r.IsAccessError(err) {
   534  		if rpc.Login() {
   535  			r.AppendToSet(keyName, value)
   536  		}
   537  	}
   538  }
   539  
   540  func (r *RPCStorageHandler) AppendToSetPipelined(key string, values []string) {
   541  	// just falls back to AppendToSet
   542  	// TODO: introduce new RPC method for pipelined operation
   543  	for _, val := range values {
   544  		r.AppendToSet(key, val)
   545  	}
   546  }
   547  
   548  // SetScrollingWindow is used in the rate limiter to handle rate limits fairly.
   549  func (r *RPCStorageHandler) SetRollingWindow(keyName string, per int64, val string, pipeline bool) (int, []interface{}) {
   550  	start := time.Now() // get current time
   551  	ibd := apidef.InboundData{
   552  		KeyName: keyName,
   553  		Per:     per,
   554  		Expire:  -1,
   555  	}
   556  
   557  	intVal, err := rpc.FuncClientSingleton("SetRollingWindow", ibd)
   558  	if err != nil {
   559  		rpc.EmitErrorEventKv(
   560  			rpc.FuncClientSingletonCall,
   561  			"SetRollingWindow",
   562  			err,
   563  			map[string]string{
   564  				"keyName": keyName,
   565  				"per":     strconv.Itoa(int(per)),
   566  			},
   567  		)
   568  
   569  		if r.IsAccessError(err) {
   570  			if rpc.Login() {
   571  				return r.SetRollingWindow(keyName, per, val, false)
   572  			}
   573  		}
   574  	}
   575  
   576  	elapsed := time.Since(start)
   577  	log.Debug("SetRollingWindow took ", elapsed)
   578  
   579  	if intVal == nil {
   580  		log.Warning("RPC Handler: SetRollingWindow() returned nil, returning 0")
   581  		return 0, nil
   582  	}
   583  
   584  	return intVal.(int), nil
   585  
   586  }
   587  
   588  func (r *RPCStorageHandler) GetRollingWindow(keyName string, per int64, pipeline bool) (int, []interface{}) {
   589  	log.Warning("Not Implemented!")
   590  	return 0, nil
   591  }
   592  
   593  func (r RPCStorageHandler) GetSet(keyName string) (map[string]string, error) {
   594  	log.Error("RPCStorageHandler.GetSet - Not implemented")
   595  	return nil, nil
   596  }
   597  
   598  func (r RPCStorageHandler) AddToSet(keyName, value string) {
   599  	log.Error("RPCStorageHandler.AddToSet - Not implemented")
   600  }
   601  
   602  func (r RPCStorageHandler) RemoveFromSet(keyName, value string) {
   603  	log.Error("RPCStorageHandler.RemoveFromSet - Not implemented")
   604  }
   605  
   606  func (r RPCStorageHandler) IsAccessError(err error) bool {
   607  	if err != nil {
   608  		return err.Error() == "Access Denied"
   609  	}
   610  	return false
   611  }
   612  
   613  // GetAPIDefinitions will pull API definitions from the RPC server
   614  func (r *RPCStorageHandler) GetApiDefinitions(orgId string, tags []string) string {
   615  	dr := apidef.DefRequest{
   616  		OrgId: orgId,
   617  		Tags:  tags,
   618  	}
   619  
   620  	defString, err := rpc.FuncClientSingleton("GetApiDefinitions", dr)
   621  	if err != nil {
   622  		rpc.EmitErrorEventKv(
   623  			rpc.FuncClientSingletonCall,
   624  			"GetApiDefinitions",
   625  			err,
   626  			map[string]string{
   627  				"orgId": orgId,
   628  				"tags":  strings.Join(tags, ","),
   629  			},
   630  		)
   631  
   632  		if r.IsAccessError(err) {
   633  			if rpc.Login() {
   634  				return r.GetApiDefinitions(orgId, tags)
   635  			}
   636  		}
   637  
   638  		return ""
   639  	}
   640  	log.Debug("API Definitions retrieved")
   641  
   642  	if defString == nil {
   643  		log.Warning("RPC Handler: GetApiDefinitions() returned nil, returning empty string")
   644  		return ""
   645  	}
   646  	return defString.(string)
   647  }
   648  
   649  // GetPolicies will pull Policies from the RPC server
   650  func (r *RPCStorageHandler) GetPolicies(orgId string) string {
   651  	defString, err := rpc.FuncClientSingleton("GetPolicies", orgId)
   652  	if err != nil {
   653  		rpc.EmitErrorEventKv(
   654  			rpc.FuncClientSingletonCall,
   655  			"GetPolicies",
   656  			err,
   657  			map[string]string{
   658  				"orgId": orgId,
   659  			},
   660  		)
   661  
   662  		if r.IsAccessError(err) {
   663  			if rpc.Login() {
   664  				return r.GetPolicies(orgId)
   665  			}
   666  		}
   667  
   668  		return ""
   669  	}
   670  
   671  	if defString != nil {
   672  		return defString.(string)
   673  	}
   674  	return ""
   675  }
   676  
   677  // CheckForReload will start a long poll
   678  func (r *RPCStorageHandler) CheckForReload(orgId string) {
   679  	log.Debug("[RPC STORE] Check Reload called...")
   680  	reload, err := rpc.FuncClientSingleton("CheckReload", orgId)
   681  	if err != nil {
   682  		rpc.EmitErrorEventKv(
   683  			rpc.FuncClientSingletonCall,
   684  			"CheckReload",
   685  			err,
   686  			map[string]string{
   687  				"orgId": orgId,
   688  			},
   689  		)
   690  		if r.IsAccessError(err) {
   691  			log.Warning("[RPC STORE] CheckReload: Not logged in")
   692  			if rpc.Login() {
   693  				r.CheckForReload(orgId)
   694  			}
   695  		} else if !strings.Contains(err.Error(), "Cannot obtain response during") {
   696  			log.Warning("[RPC STORE] RPC Reload Checker encountered unexpected error: ", err)
   697  		}
   698  
   699  		time.Sleep(1 * time.Second)
   700  	} else if reload == true {
   701  		// Do the reload!
   702  		log.Warning("[RPC STORE] Received Reload instruction!")
   703  		go func() {
   704  			MainNotifier.Notify(Notification{Command: NoticeGroupReload})
   705  		}()
   706  	}
   707  }
   708  
   709  func (r *RPCStorageHandler) StartRPCLoopCheck(orgId string) {
   710  	if config.Global().SlaveOptions.DisableKeySpaceSync {
   711  		return
   712  	}
   713  
   714  	log.Info("[RPC] Starting keyspace poller")
   715  
   716  	for {
   717  		r.CheckForKeyspaceChanges(orgId)
   718  		time.Sleep(10 * time.Second)
   719  	}
   720  }
   721  
   722  func (r *RPCStorageHandler) StartRPCKeepaliveWatcher() {
   723  	log.WithFields(logrus.Fields{
   724  		"prefix": "RPC Conn Mgr",
   725  	}).Info("[RPC Conn Mgr] Starting keepalive watcher...")
   726  	for {
   727  
   728  		if err := r.SetKey("0000", "0000", 10); err != nil {
   729  			log.WithError(err).WithFields(logrus.Fields{
   730  				"prefix": "RPC Conn Mgr",
   731  			}).Info("Can't connect to RPC layer")
   732  
   733  			if r.IsAccessError(err) {
   734  				if rpc.Login() {
   735  					continue
   736  				}
   737  			}
   738  
   739  			if strings.Contains(err.Error(), "Cannot obtain response during timeout") {
   740  				continue
   741  			}
   742  		}
   743  
   744  		time.Sleep(10 * time.Second)
   745  	}
   746  }
   747  
   748  // CheckForKeyspaceChanges will poll for keysace changes
   749  func (r *RPCStorageHandler) CheckForKeyspaceChanges(orgId string) {
   750  	log.Debug("Checking for keyspace changes...")
   751  
   752  	var keys interface{}
   753  	var err error
   754  	var funcName string
   755  	var req interface{}
   756  
   757  	reqData := map[string]string{}
   758  	if groupID := config.Global().SlaveOptions.GroupID; groupID == "" {
   759  		funcName = "GetKeySpaceUpdate"
   760  		req = orgId
   761  		reqData["orgId"] = orgId
   762  	} else {
   763  		funcName = "GetGroupKeySpaceUpdate"
   764  		req = apidef.GroupKeySpaceRequest{
   765  			OrgID:   orgId,
   766  			GroupID: groupID,
   767  		}
   768  		reqData["orgId"] = orgId
   769  		reqData["GroupID"] = groupID
   770  	}
   771  
   772  	keys, err = rpc.FuncClientSingleton(funcName, req)
   773  	if err != nil {
   774  		rpc.EmitErrorEventKv(
   775  			rpc.FuncClientSingletonCall,
   776  			funcName,
   777  			err,
   778  			reqData,
   779  		)
   780  		if r.IsAccessError(err) {
   781  			if rpc.Login() {
   782  				r.CheckForKeyspaceChanges(orgId)
   783  			}
   784  		}
   785  		log.Warning("Keyspace warning: ", err)
   786  		return
   787  	}
   788  
   789  	if keys == nil {
   790  		log.Info("Keys returned nil object, skipping check")
   791  		return
   792  	}
   793  
   794  	if len(keys.([]string)) > 0 {
   795  		log.Info("Keyspace changes detected, updating local cache")
   796  		go r.ProcessKeySpaceChanges(keys.([]string))
   797  	}
   798  }
   799  
   800  func getSessionAndCreate(keyName string, r *RPCStorageHandler) {
   801  	newKeyName := "apikey-" + storage.HashStr(keyName)
   802  	sessionString, err := r.GetRawKey(keyName)
   803  	if err != nil {
   804  		log.Error("Key not found in master - skipping")
   805  	} else {
   806  		handleAddKey(keyName, newKeyName[7:], sessionString, "-1")
   807  	}
   808  }
   809  
   810  func (r *RPCStorageHandler) ProcessKeySpaceChanges(keys []string) {
   811  	keysToReset := map[string]bool{}
   812  	TokensToBeRevoked := map[string]string{}
   813  	ClientsToBeRevoked := map[string]string{}
   814  	oauthTokenKeys := map[string]bool{}
   815  
   816  	for _, key := range keys {
   817  		splitKeys := strings.Split(key, ":")
   818  		if len(splitKeys) > 1 && splitKeys[1] == "resetQuota" {
   819  			keysToReset[splitKeys[0]] = true
   820  		} else if len(splitKeys) > 2 {
   821  			action := splitKeys[len(splitKeys)-1]
   822  			if action == "oAuthRevokeToken" || action == "oAuthRevokeAccessToken" || action == "oAuthRevokeRefreshToken" {
   823  				TokensToBeRevoked[splitKeys[0]] = key
   824  				oauthTokenKeys[key] = true
   825  			} else if action == "revoke_all_tokens" {
   826  				ClientsToBeRevoked[splitKeys[1]] = key
   827  				oauthTokenKeys[key] = true
   828  			}
   829  		}
   830  	}
   831  
   832  	for clientId, key := range ClientsToBeRevoked {
   833  		splitKeys := strings.Split(key, ":")
   834  		apiId := splitKeys[0]
   835  		clientSecret := splitKeys[2]
   836  		storage, _, err := GetStorageForApi(apiId)
   837  		if err != nil {
   838  			continue
   839  		}
   840  		_, tokens, _ := RevokeAllTokens(storage, clientId, clientSecret)
   841  		keys = append(keys, tokens...)
   842  	}
   843  
   844  	//single and specific tokens
   845  	for token, key := range TokensToBeRevoked {
   846  		//key formed as: token:apiId:tokenActionTypeHint
   847  		//but hashed as: token#hashed:apiId:tokenActionTypeHint
   848  		splitKeys := strings.Split(key, ":")
   849  		apiId := splitKeys[1]
   850  		tokenActionTypeHint := splitKeys[2]
   851  		hashedKey := strings.Contains(token, "#hashed")
   852  		if !hashedKey {
   853  			storage, _, err := GetStorageForApi(apiId)
   854  			if err != nil {
   855  				continue
   856  			}
   857  			var tokenTypeHint string
   858  			switch tokenActionTypeHint {
   859  			case "oAuthRevokeAccessToken":
   860  				tokenTypeHint = "access_token"
   861  			case "oAuthRevokeRefreshToken":
   862  				tokenTypeHint = "refresh_token"
   863  			}
   864  			RevokeToken(storage, token, tokenTypeHint)
   865  		} else {
   866  			token = strings.Split(token, "#")[0]
   867  			handleDeleteHashedKey(token, apiId, false)
   868  		}
   869  		SessionCache.Delete(token)
   870  		RPCGlobalCache.Delete(r.KeyPrefix + token)
   871  	}
   872  
   873  	for _, key := range keys {
   874  		_, isOauthTokenKey := oauthTokenKeys[key]
   875  		if !isOauthTokenKey {
   876  			splitKeys := strings.Split(key, ":")
   877  			_, resetQuota := keysToReset[splitKeys[0]]
   878  			if len(splitKeys) > 1 && splitKeys[1] == "hashed" {
   879  				key = splitKeys[0]
   880  				log.Info("--> removing cached (hashed) key: ", splitKeys[0])
   881  				handleDeleteHashedKey(splitKeys[0], "", resetQuota)
   882  				getSessionAndCreate(splitKeys[0], r)
   883  			} else {
   884  				log.Info("--> removing cached key: ", key)
   885  				handleDeleteKey(key, "-1", resetQuota)
   886  				getSessionAndCreate(splitKeys[0], r)
   887  			}
   888  			SessionCache.Delete(key)
   889  			RPCGlobalCache.Delete(r.KeyPrefix + key)
   890  		}
   891  	}
   892  	// Notify rest of gateways in cluster to flush cache
   893  	n := Notification{
   894  		Command: KeySpaceUpdateNotification,
   895  		Payload: strings.Join(keys, ","),
   896  	}
   897  	MainNotifier.Notify(n)
   898  }
   899  
   900  func (r *RPCStorageHandler) DeleteScanMatch(pattern string) bool {
   901  	log.Error("RPCStorageHandler.DeleteScanMatch - Not implemented")
   902  	return false
   903  }
   904  
   905  func (r *RPCStorageHandler) GetKeyPrefix() string {
   906  	log.Error("RPCStorageHandler.GetKeyPrefix - Not implemented")
   907  	return ""
   908  }
   909  
   910  func (r *RPCStorageHandler) AddToSortedSet(keyName, value string, score float64) {
   911  	handleGlobalAddToSortedSet(keyName, value, score)
   912  }
   913  
   914  func (r *RPCStorageHandler) GetSortedSetRange(keyName, scoreFrom, scoreTo string) ([]string, []float64, error) {
   915  	return handleGetSortedSetRange(keyName, scoreFrom, scoreTo)
   916  }
   917  
   918  func (r *RPCStorageHandler) RemoveSortedSetRange(keyName, scoreFrom, scoreTo string) error {
   919  	return handleRemoveSortedSetRange(keyName, scoreFrom, scoreTo)
   920  }
   921  
   922  func (r *RPCStorageHandler) RemoveFromList(keyName, value string) error {
   923  	log.Error("Not implemented")
   924  	return nil
   925  }
   926  
   927  func (r *RPCStorageHandler) GetListRange(keyName string, from, to int64) ([]string, error) {
   928  	log.Error("Not implemented")
   929  	return nil, nil
   930  }
   931  
   932  func (r *RPCStorageHandler) Exists(keyName string) (bool, error) {
   933  	log.Error("Not implemented")
   934  	return false, nil
   935  }