github.com/Tyktechnologies/tyk@v2.9.5+incompatible/gateway/api_definition.go (about)

     1  package gateway
     2  
     3  import (
     4  	"encoding/base64"
     5  	"encoding/json"
     6  	"errors"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"net/http"
    11  	"net/url"
    12  	"os"
    13  	"path/filepath"
    14  	"strings"
    15  	"sync"
    16  	"sync/atomic"
    17  	"text/template"
    18  	"time"
    19  
    20  	"github.com/cenk/backoff"
    21  
    22  	"gopkg.in/Masterminds/sprig.v2"
    23  
    24  	"github.com/TykTechnologies/tyk/headers"
    25  	"github.com/TykTechnologies/tyk/rpc"
    26  	"github.com/gorilla/mux"
    27  
    28  	"github.com/sirupsen/logrus"
    29  
    30  	"github.com/TykTechnologies/circuitbreaker"
    31  	"github.com/TykTechnologies/gojsonschema"
    32  	"github.com/TykTechnologies/tyk/apidef"
    33  	"github.com/TykTechnologies/tyk/config"
    34  	"github.com/TykTechnologies/tyk/regexp"
    35  	"github.com/TykTechnologies/tyk/storage"
    36  )
    37  
    38  //const used by cache middleware
    39  const SAFE_METHODS = "SAFE_METHODS"
    40  
    41  const (
    42  	LDAPStorageEngine apidef.StorageEngineCode = "ldap"
    43  	RPCStorageEngine  apidef.StorageEngineCode = "rpc"
    44  )
    45  
    46  // Constants used by the version check middleware
    47  const (
    48  	headerLocation    = "header"
    49  	urlParamLocation  = "url-param"
    50  	urlLocation       = "url"
    51  	expiredTimeFormat = "2006-01-02 15:04"
    52  )
    53  
    54  // URLStatus is a custom enum type to avoid collisions
    55  type URLStatus int
    56  
    57  // Enums representing the various statuses for a VersionInfo Path match during a
    58  // proxy request
    59  const (
    60  	_ URLStatus = iota
    61  	Ignored
    62  	WhiteList
    63  	BlackList
    64  	Cached
    65  	Transformed
    66  	TransformedJQ
    67  	HeaderInjected
    68  	HeaderInjectedResponse
    69  	TransformedResponse
    70  	TransformedJQResponse
    71  	HardTimeout
    72  	CircuitBreaker
    73  	URLRewrite
    74  	VirtualPath
    75  	RequestSizeLimit
    76  	MethodTransformed
    77  	RequestTracked
    78  	RequestNotTracked
    79  	ValidateJSONRequest
    80  	Internal
    81  )
    82  
    83  // RequestStatus is a custom type to avoid collisions
    84  type RequestStatus string
    85  
    86  // Statuses of the request, all are false-y except StatusOk and StatusOkAndIgnore
    87  const (
    88  	VersionNotFound                RequestStatus = "Version information not found"
    89  	VersionDoesNotExist            RequestStatus = "This API version does not seem to exist"
    90  	VersionWhiteListStatusNotFound RequestStatus = "WhiteListStatus for path not found"
    91  	VersionExpired                 RequestStatus = "Api Version has expired, please check documentation or contact administrator"
    92  	EndPointNotAllowed             RequestStatus = "Requested endpoint is forbidden"
    93  	StatusOkAndIgnore              RequestStatus = "Everything OK, passing and not filtering"
    94  	StatusOk                       RequestStatus = "Everything OK, passing"
    95  	StatusCached                   RequestStatus = "Cached path"
    96  	StatusTransform                RequestStatus = "Transformed path"
    97  	StatusTransformResponse        RequestStatus = "Transformed response"
    98  	StatusTransformJQ              RequestStatus = "Transformed path with JQ"
    99  	StatusTransformJQResponse      RequestStatus = "Transformed response with JQ"
   100  	StatusHeaderInjected           RequestStatus = "Header injected"
   101  	StatusMethodTransformed        RequestStatus = "Method Transformed"
   102  	StatusHeaderInjectedResponse   RequestStatus = "Header injected on response"
   103  	StatusRedirectFlowByReply      RequestStatus = "Exceptional action requested, redirecting flow!"
   104  	StatusHardTimeout              RequestStatus = "Hard Timeout enforced on path"
   105  	StatusCircuitBreaker           RequestStatus = "Circuit breaker enforced"
   106  	StatusURLRewrite               RequestStatus = "URL Rewritten"
   107  	StatusVirtualPath              RequestStatus = "Virtual Endpoint"
   108  	StatusRequestSizeControlled    RequestStatus = "Request Size Limited"
   109  	StatusRequestTracked           RequestStatus = "Request Tracked"
   110  	StatusRequestNotTracked        RequestStatus = "Request Not Tracked"
   111  	StatusValidateJSON             RequestStatus = "Validate JSON"
   112  	StatusInternal                 RequestStatus = "Internal path"
   113  )
   114  
   115  // URLSpec represents a flattened specification for URLs, used to check if a proxy URL
   116  // path is on any of the white, black or ignored lists. This is generated as part of the
   117  // configuration init
   118  type URLSpec struct {
   119  	Spec                      *regexp.Regexp
   120  	Status                    URLStatus
   121  	MethodActions             map[string]apidef.EndpointMethodMeta
   122  	CacheConfig               EndPointCacheMeta
   123  	TransformAction           TransformSpec
   124  	TransformResponseAction   TransformSpec
   125  	TransformJQAction         TransformJQSpec
   126  	TransformJQResponseAction TransformJQSpec
   127  	InjectHeaders             apidef.HeaderInjectionMeta
   128  	InjectHeadersResponse     apidef.HeaderInjectionMeta
   129  	HardTimeout               apidef.HardTimeoutMeta
   130  	CircuitBreaker            ExtendedCircuitBreakerMeta
   131  	URLRewrite                *apidef.URLRewriteMeta
   132  	VirtualPathSpec           apidef.VirtualMeta
   133  	RequestSize               apidef.RequestSizeMeta
   134  	MethodTransform           apidef.MethodTransformMeta
   135  	TrackEndpoint             apidef.TrackEndpointMeta
   136  	DoNotTrackEndpoint        apidef.TrackEndpointMeta
   137  	ValidatePathMeta          apidef.ValidatePathMeta
   138  	Internal                  apidef.InternalMeta
   139  	IgnoreCase                bool
   140  }
   141  
   142  type EndPointCacheMeta struct {
   143  	Method                 string
   144  	CacheKeyRegex          string
   145  	CacheOnlyResponseCodes []int
   146  }
   147  
   148  type TransformSpec struct {
   149  	apidef.TemplateMeta
   150  	Template *template.Template
   151  }
   152  
   153  type ExtendedCircuitBreakerMeta struct {
   154  	apidef.CircuitBreakerMeta
   155  	CB *circuit.Breaker `json:"-"`
   156  }
   157  
   158  // APISpec represents a path specification for an API, to avoid enumerating multiple nested lists, a single
   159  // flattened URL list is checked for matching paths and then it's status evaluated if found.
   160  type APISpec struct {
   161  	*apidef.APIDefinition
   162  	sync.RWMutex
   163  
   164  	RxPaths                  map[string][]URLSpec
   165  	WhiteListEnabled         map[string]bool
   166  	target                   *url.URL
   167  	AuthManager              SessionHandler
   168  	SessionManager           SessionHandler
   169  	OAuthManager             *OAuthManager
   170  	OrgSessionManager        SessionHandler
   171  	EventPaths               map[apidef.TykEvent][]config.TykEventHandler
   172  	Health                   HealthChecker
   173  	JSVM                     JSVM
   174  	ResponseChain            []TykResponseHandler
   175  	RoundRobin               RoundRobin
   176  	URLRewriteEnabled        bool
   177  	CircuitBreakerEnabled    bool
   178  	EnforcedTimeoutEnabled   bool
   179  	LastGoodHostList         *apidef.HostList
   180  	HasRun                   bool
   181  	ServiceRefreshInProgress bool
   182  	HTTPTransport            http.RoundTripper
   183  	HTTPTransportCreated     time.Time
   184  	WSTransport              http.RoundTripper
   185  	WSTransportCreated       time.Time
   186  	GlobalConfig             config.Config
   187  	OrgHasNoSession          bool
   188  
   189  	middlewareChain *ChainObject
   190  
   191  	network NetworkStats
   192  }
   193  
   194  // Release re;leases all resources associated with API spec
   195  func (s *APISpec) Release() {
   196  	// release circuit breaker resources
   197  	for _, path := range s.RxPaths {
   198  		for _, urlSpec := range path {
   199  			if urlSpec.CircuitBreaker.CB != nil {
   200  				// this will force CB-event reading Go-routine and subscriber Go-routine to exit
   201  				urlSpec.CircuitBreaker.CB.Stop()
   202  			}
   203  		}
   204  	}
   205  
   206  	// release all other resources associated with spec
   207  }
   208  
   209  // Validate returns nil if s is a valid spec and an error stating why the spec is not valid.
   210  func (s *APISpec) Validate() error {
   211  	// For tcp services we need to make sure we can bind to the port.
   212  	switch s.Protocol {
   213  	case "tcp", "tls":
   214  		return s.validateTCP()
   215  	default:
   216  		return s.validateHTTP()
   217  	}
   218  }
   219  
   220  func (s *APISpec) validateTCP() error {
   221  	if s.ListenPort == 0 {
   222  		return errors.New("missing listening port")
   223  	}
   224  	return nil
   225  }
   226  
   227  func (s *APISpec) validateHTTP() error {
   228  	// NOOP
   229  	return nil
   230  }
   231  
   232  // APIDefinitionLoader will load an Api definition from a storage
   233  // system.
   234  type APIDefinitionLoader struct{}
   235  
   236  // Nonce to use when interacting with the dashboard service
   237  var ServiceNonce string
   238  
   239  // MakeSpec will generate a flattened URLSpec from and APIDefinitions' VersionInfo data. paths are
   240  // keyed to the Api version name, which is determined during routing to speed up lookups
   241  func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition, logger *logrus.Entry) *APISpec {
   242  	spec := &APISpec{}
   243  
   244  	if logger == nil {
   245  		logger = logrus.NewEntry(log)
   246  	}
   247  
   248  	// parse version expiration time stamps
   249  	for key, ver := range def.VersionData.Versions {
   250  		if ver.Expires == "" || ver.Expires == "-1" {
   251  			continue
   252  		}
   253  		// calculate the time
   254  		if t, err := time.Parse(expiredTimeFormat, ver.Expires); err != nil {
   255  			logger.WithError(err).WithField("Expires", ver.Expires).Error("Could not parse expiry date for API")
   256  		} else {
   257  			ver.ExpiresTs = t
   258  			def.VersionData.Versions[key] = ver
   259  		}
   260  	}
   261  
   262  	spec.APIDefinition = def
   263  
   264  	// We'll push the default HealthChecker:
   265  	spec.Health = &DefaultHealthChecker{
   266  		APIID: spec.APIID,
   267  	}
   268  
   269  	// Add any new session managers or auth handlers here
   270  	spec.AuthManager = &DefaultSessionManager{
   271  		orgID: spec.OrgID,
   272  	}
   273  
   274  	spec.SessionManager = &DefaultSessionManager{
   275  		orgID: spec.OrgID,
   276  	}
   277  	spec.OrgSessionManager = &DefaultSessionManager{
   278  		orgID: spec.OrgID,
   279  	}
   280  
   281  	spec.GlobalConfig = config.Global()
   282  
   283  	// Create and init the virtual Machine
   284  	if config.Global().EnableJSVM {
   285  		mwPaths, _, _, _, _, _ := loadCustomMiddleware(spec)
   286  
   287  		hasVirtualEndpoint := false
   288  
   289  		for _, version := range spec.VersionData.Versions {
   290  			if len(version.ExtendedPaths.Virtual) > 0 {
   291  				hasVirtualEndpoint = true
   292  				break
   293  			}
   294  		}
   295  
   296  		if spec.CustomMiddlewareBundle != "" || len(mwPaths) > 0 || hasVirtualEndpoint {
   297  			spec.JSVM.Init(spec, logger)
   298  		}
   299  	}
   300  
   301  	// Set up Event Handlers
   302  	if len(def.EventHandlers.Events) > 0 {
   303  		logger.Debug("Initializing event handlers")
   304  	}
   305  	spec.EventPaths = make(map[apidef.TykEvent][]config.TykEventHandler)
   306  	for eventName, eventHandlerConfs := range def.EventHandlers.Events {
   307  		logger.Debug("FOUND EVENTS TO INIT")
   308  		for _, handlerConf := range eventHandlerConfs {
   309  			logger.Debug("CREATING EVENT HANDLERS")
   310  			eventHandlerInstance, err := EventHandlerByName(handlerConf, spec)
   311  
   312  			if err != nil {
   313  				logger.Error("Failed to init event handler: ", err)
   314  			} else {
   315  				logger.Debug("Init Event Handler: ", eventName)
   316  				spec.EventPaths[eventName] = append(spec.EventPaths[eventName], eventHandlerInstance)
   317  			}
   318  
   319  		}
   320  	}
   321  
   322  	spec.RxPaths = make(map[string][]URLSpec, len(def.VersionData.Versions))
   323  	spec.WhiteListEnabled = make(map[string]bool, len(def.VersionData.Versions))
   324  	for _, v := range def.VersionData.Versions {
   325  		var pathSpecs []URLSpec
   326  		var whiteListSpecs bool
   327  
   328  		// If we have transitioned to extended path specifications, we should use these now
   329  		if v.UseExtendedPaths {
   330  			pathSpecs, whiteListSpecs = a.getExtendedPathSpecs(v, spec)
   331  		} else {
   332  			logger.Warning("Legacy path detected! Upgrade to extended.")
   333  			pathSpecs, whiteListSpecs = a.getPathSpecs(v)
   334  		}
   335  		spec.RxPaths[v.Name] = pathSpecs
   336  		spec.WhiteListEnabled[v.Name] = whiteListSpecs
   337  	}
   338  
   339  	return spec
   340  }
   341  
   342  // FromDashboardService will connect and download ApiDefintions from a Tyk Dashboard instance.
   343  func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) ([]*APISpec, error) {
   344  	// Get the definitions
   345  	log.Debug("Calling: ", endpoint)
   346  	newRequest, err := http.NewRequest("GET", endpoint, nil)
   347  	if err != nil {
   348  		log.Error("Failed to create request: ", err)
   349  	}
   350  
   351  	newRequest.Header.Set("authorization", secret)
   352  	log.Debug("Using: NodeID: ", GetNodeID())
   353  	newRequest.Header.Set(headers.XTykNodeID, GetNodeID())
   354  
   355  	newRequest.Header.Set(headers.XTykNonce, ServiceNonce)
   356  
   357  	c := initialiseClient()
   358  	resp, err := c.Do(newRequest)
   359  	if err != nil {
   360  		return nil, err
   361  	}
   362  	defer resp.Body.Close()
   363  
   364  	if resp.StatusCode == http.StatusForbidden {
   365  		body, _ := ioutil.ReadAll(resp.Body)
   366  		reLogin()
   367  		return nil, fmt.Errorf("login failure, Response was: %v", string(body))
   368  	}
   369  
   370  	if resp.StatusCode != http.StatusOK {
   371  		body, _ := ioutil.ReadAll(resp.Body)
   372  		reLogin()
   373  		return nil, fmt.Errorf("dashboard API error, response was: %v", string(body))
   374  	}
   375  
   376  	// Extract tagged APIs#
   377  	var list struct {
   378  		Message []struct {
   379  			ApiDefinition *apidef.APIDefinition `bson:"api_definition" json:"api_definition"`
   380  		}
   381  		Nonce string
   382  	}
   383  	if err := json.NewDecoder(resp.Body).Decode(&list); err != nil {
   384  		body, _ := ioutil.ReadAll(resp.Body)
   385  		return nil, fmt.Errorf("failed to decode body: %v body was: %v", err, string(body))
   386  	}
   387  
   388  	// Extract tagged entries only
   389  	apiDefs := make([]*apidef.APIDefinition, 0)
   390  
   391  	if config.Global().DBAppConfOptions.NodeIsSegmented {
   392  		tagList := make(map[string]bool, len(config.Global().DBAppConfOptions.Tags))
   393  		toLoad := make(map[string]*apidef.APIDefinition)
   394  
   395  		for _, mt := range config.Global().DBAppConfOptions.Tags {
   396  			tagList[mt] = true
   397  		}
   398  
   399  		for _, apiEntry := range list.Message {
   400  			for _, t := range apiEntry.ApiDefinition.Tags {
   401  				if tagList[t] {
   402  					toLoad[apiEntry.ApiDefinition.APIID] = apiEntry.ApiDefinition
   403  				}
   404  			}
   405  		}
   406  
   407  		for _, apiDef := range toLoad {
   408  			apiDefs = append(apiDefs, apiDef)
   409  		}
   410  	} else {
   411  		for _, apiEntry := range list.Message {
   412  			apiDefs = append(apiDefs, apiEntry.ApiDefinition)
   413  		}
   414  	}
   415  
   416  	// Process
   417  	var specs []*APISpec
   418  	for _, def := range apiDefs {
   419  		spec := a.MakeSpec(def, nil)
   420  		specs = append(specs, spec)
   421  	}
   422  
   423  	// Set the nonce
   424  	ServiceNonce = list.Nonce
   425  	log.Debug("Loading APIS Finished: Nonce Set: ", ServiceNonce)
   426  
   427  	return specs, nil
   428  }
   429  
   430  // FromCloud will connect and download ApiDefintions from a Mongo DB instance.
   431  func (a APIDefinitionLoader) FromRPC(orgId string) ([]*APISpec, error) {
   432  	if rpc.IsEmergencyMode() {
   433  		return LoadDefinitionsFromRPCBackup()
   434  	}
   435  
   436  	store := RPCStorageHandler{}
   437  	if !store.Connect() {
   438  		return nil, errors.New("Can't connect RPC layer")
   439  	}
   440  
   441  	// enable segments
   442  	var tags []string
   443  	if config.Global().DBAppConfOptions.NodeIsSegmented {
   444  		log.Info("Segmented node, loading: ", config.Global().DBAppConfOptions.Tags)
   445  		tags = config.Global().DBAppConfOptions.Tags
   446  	}
   447  
   448  	apiCollection := store.GetApiDefinitions(orgId, tags)
   449  
   450  	//store.Disconnect()
   451  
   452  	if rpc.LoadCount() > 0 {
   453  		if err := saveRPCDefinitionsBackup(apiCollection); err != nil {
   454  			return nil, err
   455  		}
   456  	}
   457  
   458  	return a.processRPCDefinitions(apiCollection)
   459  }
   460  
   461  func (a APIDefinitionLoader) processRPCDefinitions(apiCollection string) ([]*APISpec, error) {
   462  
   463  	var apiDefs []*apidef.APIDefinition
   464  	if err := json.Unmarshal([]byte(apiCollection), &apiDefs); err != nil {
   465  		return nil, err
   466  	}
   467  
   468  	var specs []*APISpec
   469  	for _, def := range apiDefs {
   470  		def.DecodeFromDB()
   471  
   472  		if config.Global().SlaveOptions.BindToSlugsInsteadOfListenPaths {
   473  			newListenPath := "/" + def.Slug //+ "/"
   474  			log.Warning("Binding to ",
   475  				newListenPath,
   476  				" instead of ",
   477  				def.Proxy.ListenPath)
   478  
   479  			def.Proxy.ListenPath = newListenPath
   480  		}
   481  
   482  		spec := a.MakeSpec(def, nil)
   483  		specs = append(specs, spec)
   484  	}
   485  
   486  	return specs, nil
   487  }
   488  
   489  func (a APIDefinitionLoader) ParseDefinition(r io.Reader) *apidef.APIDefinition {
   490  	def := &apidef.APIDefinition{}
   491  	if err := json.NewDecoder(r).Decode(def); err != nil {
   492  		log.Error("[RPC] --> Couldn't unmarshal api configuration: ", err)
   493  	}
   494  	return def
   495  }
   496  
   497  // FromDir will load APIDefinitions from a directory on the filesystem. Definitions need
   498  // to be the JSON representation of APIDefinition object
   499  func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
   500  	var specs []*APISpec
   501  	// Grab json files from directory
   502  	paths, _ := filepath.Glob(filepath.Join(dir, "*.json"))
   503  	for _, path := range paths {
   504  		log.Info("Loading API Specification from ", path)
   505  		f, err := os.Open(path)
   506  		if err != nil {
   507  			log.Error("Couldn't open api configuration file: ", err)
   508  			continue
   509  		}
   510  		def := a.ParseDefinition(f)
   511  		f.Close()
   512  		spec := a.MakeSpec(def, nil)
   513  		specs = append(specs, spec)
   514  	}
   515  	return specs
   516  }
   517  
   518  func (a APIDefinitionLoader) getPathSpecs(apiVersionDef apidef.VersionInfo) ([]URLSpec, bool) {
   519  	ignoredPaths := a.compilePathSpec(apiVersionDef.Paths.Ignored, Ignored)
   520  	blackListPaths := a.compilePathSpec(apiVersionDef.Paths.BlackList, BlackList)
   521  	whiteListPaths := a.compilePathSpec(apiVersionDef.Paths.WhiteList, WhiteList)
   522  
   523  	combinedPath := []URLSpec{}
   524  	combinedPath = append(combinedPath, ignoredPaths...)
   525  	combinedPath = append(combinedPath, blackListPaths...)
   526  	combinedPath = append(combinedPath, whiteListPaths...)
   527  
   528  	return combinedPath, len(whiteListPaths) > 0
   529  }
   530  
   531  func (a APIDefinitionLoader) generateRegex(stringSpec string, newSpec *URLSpec, specType URLStatus) {
   532  	apiLangIDsRegex := regexp.MustCompile(`{([^}]*)}`)
   533  	asRegexStr := apiLangIDsRegex.ReplaceAllString(stringSpec, `([^/]*)`)
   534  	// Case insensitive match
   535  	if newSpec.IgnoreCase || config.Global().IgnoreEndpointCase {
   536  		asRegexStr = "(?i)" + asRegexStr
   537  	}
   538  	asRegex, _ := regexp.Compile(asRegexStr)
   539  	newSpec.Status = specType
   540  	newSpec.Spec = asRegex
   541  }
   542  
   543  func (a APIDefinitionLoader) compilePathSpec(paths []string, specType URLStatus) []URLSpec {
   544  	// transform a configuration URL into an array of URLSpecs
   545  	// This way we can iterate the whole array once, on match we break with status
   546  	urlSpec := []URLSpec{}
   547  
   548  	for _, stringSpec := range paths {
   549  		newSpec := URLSpec{}
   550  		a.generateRegex(stringSpec, &newSpec, specType)
   551  		urlSpec = append(urlSpec, newSpec)
   552  	}
   553  
   554  	return urlSpec
   555  }
   556  
   557  func (a APIDefinitionLoader) compileExtendedPathSpec(ignoreEndpointCase bool, paths []apidef.EndPointMeta, specType URLStatus) []URLSpec {
   558  	// transform an extended configuration URL into an array of URLSpecs
   559  	// This way we can iterate the whole array once, on match we break with status
   560  	urlSpec := []URLSpec{}
   561  
   562  	for _, stringSpec := range paths {
   563  		newSpec := URLSpec{IgnoreCase: stringSpec.IgnoreCase || ignoreEndpointCase}
   564  		a.generateRegex(stringSpec.Path, &newSpec, specType)
   565  
   566  		// Extend with method actions
   567  		newSpec.MethodActions = stringSpec.MethodActions
   568  		urlSpec = append(urlSpec, newSpec)
   569  	}
   570  
   571  	return urlSpec
   572  }
   573  
   574  func (a APIDefinitionLoader) compileCachedPathSpec(oldpaths []string, newpaths []apidef.CacheMeta) []URLSpec {
   575  	// transform an extended configuration URL into an array of URLSpecs
   576  	// This way we can iterate the whole array once, on match we break with status
   577  	urlSpec := []URLSpec{}
   578  
   579  	for _, stringSpec := range oldpaths {
   580  		newSpec := URLSpec{}
   581  		a.generateRegex(stringSpec, &newSpec, Cached)
   582  		newSpec.CacheConfig.Method = SAFE_METHODS
   583  		newSpec.CacheConfig.CacheKeyRegex = ""
   584  		// Extend with method actions
   585  		urlSpec = append(urlSpec, newSpec)
   586  	}
   587  
   588  	for _, spec := range newpaths {
   589  		newSpec := URLSpec{}
   590  		a.generateRegex(spec.Path, &newSpec, Cached)
   591  		newSpec.CacheConfig.Method = spec.Method
   592  		newSpec.CacheConfig.CacheKeyRegex = spec.CacheKeyRegex
   593  		newSpec.CacheConfig.CacheOnlyResponseCodes = spec.CacheOnlyResponseCodes
   594  		// Extend with method actions
   595  		urlSpec = append(urlSpec, newSpec)
   596  	}
   597  
   598  	return urlSpec
   599  }
   600  
   601  func (a APIDefinitionLoader) filterSprigFuncs() template.FuncMap {
   602  	tmp := sprig.GenericFuncMap()
   603  	delete(tmp, "env")
   604  	delete(tmp, "expandenv")
   605  
   606  	return template.FuncMap(tmp)
   607  }
   608  
   609  func (a APIDefinitionLoader) loadFileTemplate(path string) (*template.Template, error) {
   610  	log.Debug("-- Loading template: ", path)
   611  	return apidef.Template.New("").Funcs(a.filterSprigFuncs()).ParseFiles(path)
   612  }
   613  
   614  func (a APIDefinitionLoader) loadBlobTemplate(blob string) (*template.Template, error) {
   615  	log.Debug("-- Loading blob")
   616  	uDec, err := base64.StdEncoding.DecodeString(blob)
   617  	if err != nil {
   618  		return nil, err
   619  	}
   620  	return apidef.Template.New("").Funcs(a.filterSprigFuncs()).Parse(string(uDec))
   621  }
   622  
   623  func (a APIDefinitionLoader) compileTransformPathSpec(paths []apidef.TemplateMeta, stat URLStatus) []URLSpec {
   624  	// transform an extended configuration URL into an array of URLSpecs
   625  	// This way we can iterate the whole array once, on match we break with status
   626  	urlSpec := []URLSpec{}
   627  
   628  	log.Debug("Checking for transform paths...")
   629  	for _, stringSpec := range paths {
   630  		log.Debug("-- Generating path")
   631  		newSpec := URLSpec{}
   632  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   633  		// Extend with template actions
   634  
   635  		newTransformSpec := TransformSpec{TemplateMeta: stringSpec}
   636  
   637  		// Load the templates
   638  		var err error
   639  
   640  		switch stringSpec.TemplateData.Mode {
   641  		case apidef.UseFile:
   642  			log.Debug("-- Using File mode")
   643  			newTransformSpec.Template, err = a.loadFileTemplate(stringSpec.TemplateData.TemplateSource)
   644  		case apidef.UseBlob:
   645  			log.Debug("-- Blob mode")
   646  			newTransformSpec.Template, err = a.loadBlobTemplate(stringSpec.TemplateData.TemplateSource)
   647  		default:
   648  			log.Warning("[Transform Templates] No template mode defined! Found: ", stringSpec.TemplateData.Mode)
   649  			err = errors.New("No valid template mode defined, must be either 'file' or 'blob'")
   650  		}
   651  
   652  		if stat == Transformed {
   653  			newSpec.TransformAction = newTransformSpec
   654  		} else {
   655  			newSpec.TransformResponseAction = newTransformSpec
   656  		}
   657  
   658  		if err == nil {
   659  			urlSpec = append(urlSpec, newSpec)
   660  			log.Debug("-- Loaded")
   661  		} else {
   662  			log.Error("Template load failure! Skipping transformation: ", err)
   663  		}
   664  
   665  	}
   666  
   667  	return urlSpec
   668  }
   669  
   670  func (a APIDefinitionLoader) compileInjectedHeaderSpec(paths []apidef.HeaderInjectionMeta, stat URLStatus) []URLSpec {
   671  	// transform an extended configuration URL into an array of URLSpecs
   672  	// This way we can iterate the whole array once, on match we break with status
   673  	urlSpec := []URLSpec{}
   674  
   675  	for _, stringSpec := range paths {
   676  		newSpec := URLSpec{}
   677  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   678  		// Extend with method actions
   679  		if stat == HeaderInjected {
   680  			newSpec.InjectHeaders = stringSpec
   681  		} else {
   682  			newSpec.InjectHeadersResponse = stringSpec
   683  		}
   684  
   685  		urlSpec = append(urlSpec, newSpec)
   686  	}
   687  
   688  	return urlSpec
   689  }
   690  
   691  func (a APIDefinitionLoader) compileMethodTransformSpec(paths []apidef.MethodTransformMeta, stat URLStatus) []URLSpec {
   692  	// transform an extended configuration URL into an array of URLSpecs
   693  	// This way we can iterate the whole array once, on match we break with status
   694  	urlSpec := []URLSpec{}
   695  
   696  	for _, stringSpec := range paths {
   697  		newSpec := URLSpec{}
   698  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   699  		newSpec.MethodTransform = stringSpec
   700  
   701  		urlSpec = append(urlSpec, newSpec)
   702  	}
   703  
   704  	return urlSpec
   705  }
   706  
   707  func (a APIDefinitionLoader) compileTimeoutPathSpec(paths []apidef.HardTimeoutMeta, stat URLStatus) []URLSpec {
   708  	// transform an extended configuration URL into an array of URLSpecs
   709  	// This way we can iterate the whole array once, on match we break with status
   710  	urlSpec := []URLSpec{}
   711  
   712  	for _, stringSpec := range paths {
   713  		newSpec := URLSpec{}
   714  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   715  		// Extend with method actions
   716  		newSpec.HardTimeout = stringSpec
   717  
   718  		urlSpec = append(urlSpec, newSpec)
   719  	}
   720  
   721  	return urlSpec
   722  }
   723  
   724  func (a APIDefinitionLoader) compileRequestSizePathSpec(paths []apidef.RequestSizeMeta, stat URLStatus) []URLSpec {
   725  	// transform an extended configuration URL into an array of URLSpecs
   726  	// This way we can iterate the whole array once, on match we break with status
   727  	urlSpec := []URLSpec{}
   728  
   729  	for _, stringSpec := range paths {
   730  		newSpec := URLSpec{}
   731  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   732  		// Extend with method actions
   733  		newSpec.RequestSize = stringSpec
   734  
   735  		urlSpec = append(urlSpec, newSpec)
   736  	}
   737  
   738  	return urlSpec
   739  }
   740  
   741  func (a APIDefinitionLoader) compileCircuitBreakerPathSpec(paths []apidef.CircuitBreakerMeta, stat URLStatus, apiSpec *APISpec) []URLSpec {
   742  	// transform an extended configuration URL into an array of URLSpecs
   743  	// This way we can iterate the whole array once, on match we break with status
   744  	urlSpec := []URLSpec{}
   745  
   746  	for _, stringSpec := range paths {
   747  		newSpec := URLSpec{}
   748  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   749  		// Extend with method actions
   750  		newSpec.CircuitBreaker = ExtendedCircuitBreakerMeta{CircuitBreakerMeta: stringSpec}
   751  		log.Debug("Initialising circuit breaker for: ", stringSpec.Path)
   752  		newSpec.CircuitBreaker.CB = circuit.NewRateBreaker(stringSpec.ThresholdPercent, stringSpec.Samples)
   753  
   754  		// override backoff algorithm when is not desired to recheck the upstream before the ReturnToServiceAfter happens
   755  		if stringSpec.DisableHalfOpenState {
   756  			newSpec.CircuitBreaker.CB.BackOff = &backoff.StopBackOff{}
   757  		}
   758  
   759  		events := newSpec.CircuitBreaker.CB.Subscribe()
   760  		go func(path string, spec *APISpec, breakerPtr *circuit.Breaker) {
   761  			timerActive := false
   762  			for e := range events {
   763  				switch e {
   764  				case circuit.BreakerTripped:
   765  					log.Warning("[PROXY] [CIRCUIT BREAKER] Breaker tripped for path: ", path)
   766  					log.Debug("Breaker tripped: ", e)
   767  					// Start a timer function
   768  
   769  					if !timerActive {
   770  						go func(timeout int, breaker *circuit.Breaker) {
   771  							log.Debug("-- Sleeping for (s): ", timeout)
   772  							time.Sleep(time.Duration(timeout) * time.Second)
   773  							log.Debug("-- Resetting breaker")
   774  							breaker.Reset()
   775  							timerActive = false
   776  						}(newSpec.CircuitBreaker.ReturnToServiceAfter, breakerPtr)
   777  						timerActive = true
   778  					}
   779  
   780  					if spec.Proxy.ServiceDiscovery.UseDiscoveryService {
   781  						if ServiceCache != nil {
   782  							log.Warning("[PROXY] [CIRCUIT BREAKER] Refreshing host list")
   783  							ServiceCache.Delete(spec.APIID)
   784  						}
   785  					}
   786  
   787  					spec.FireEvent(EventBreakerTriggered, EventCurcuitBreakerMeta{
   788  						EventMetaDefault: EventMetaDefault{Message: "Breaker Tripped"},
   789  						CircuitEvent:     e,
   790  						Path:             path,
   791  						APIID:            spec.APIID,
   792  					})
   793  
   794  				case circuit.BreakerReset:
   795  					spec.FireEvent(EventBreakerTriggered, EventCurcuitBreakerMeta{
   796  						EventMetaDefault: EventMetaDefault{Message: "Breaker Reset"},
   797  						CircuitEvent:     e,
   798  						Path:             path,
   799  						APIID:            spec.APIID,
   800  					})
   801  
   802  				case circuit.BreakerStop:
   803  					// time to stop this Go-routine
   804  					return
   805  				}
   806  			}
   807  		}(stringSpec.Path, apiSpec, newSpec.CircuitBreaker.CB)
   808  
   809  		urlSpec = append(urlSpec, newSpec)
   810  	}
   811  
   812  	return urlSpec
   813  }
   814  
   815  func (a APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewriteMeta, stat URLStatus) []URLSpec {
   816  	// transform an extended configuration URL into an array of URLSpecs
   817  	// This way we can iterate the whole array once, on match we break with status
   818  	urlSpec := []URLSpec{}
   819  
   820  	for _, stringSpec := range paths {
   821  		curStringSpec := stringSpec
   822  		newSpec := URLSpec{}
   823  		a.generateRegex(curStringSpec.Path, &newSpec, stat)
   824  		// Extend with method actions
   825  		newSpec.URLRewrite = &curStringSpec
   826  
   827  		urlSpec = append(urlSpec, newSpec)
   828  	}
   829  
   830  	return urlSpec
   831  }
   832  
   833  func (a APIDefinitionLoader) compileVirtualPathspathSpec(paths []apidef.VirtualMeta, stat URLStatus, apiSpec *APISpec) []URLSpec {
   834  	if !config.Global().EnableJSVM {
   835  		return nil
   836  	}
   837  
   838  	// transform an extended configuration URL into an array of URLSpecs
   839  	// This way we can iterate the whole array once, on match we break with status
   840  	urlSpec := []URLSpec{}
   841  	for _, stringSpec := range paths {
   842  		newSpec := URLSpec{}
   843  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   844  		// Extend with method actions
   845  		newSpec.VirtualPathSpec = stringSpec
   846  
   847  		preLoadVirtualMetaCode(&newSpec.VirtualPathSpec, &apiSpec.JSVM)
   848  
   849  		urlSpec = append(urlSpec, newSpec)
   850  	}
   851  
   852  	return urlSpec
   853  }
   854  
   855  func (a APIDefinitionLoader) compileTrackedEndpointPathspathSpec(paths []apidef.TrackEndpointMeta, stat URLStatus) []URLSpec {
   856  	urlSpec := []URLSpec{}
   857  
   858  	for _, stringSpec := range paths {
   859  		newSpec := URLSpec{}
   860  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   861  
   862  		// set Path if it wasn't set
   863  		if stringSpec.Path == "" {
   864  			// even if it is empty (and regex matches everything) some middlewares expect to be value here
   865  			stringSpec.Path = "/"
   866  		}
   867  
   868  		// Extend with method actions
   869  		newSpec.TrackEndpoint = stringSpec
   870  		urlSpec = append(urlSpec, newSpec)
   871  	}
   872  
   873  	return urlSpec
   874  }
   875  
   876  func (a APIDefinitionLoader) compileValidateJSONPathspathSpec(paths []apidef.ValidatePathMeta, stat URLStatus) []URLSpec {
   877  	urlSpec := make([]URLSpec, len(paths))
   878  
   879  	for i, stringSpec := range paths {
   880  		newSpec := URLSpec{}
   881  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   882  		// Extend with method actions
   883  
   884  		stringSpec.SchemaCache = gojsonschema.NewGoLoader(stringSpec.Schema)
   885  		newSpec.ValidatePathMeta = stringSpec
   886  		urlSpec[i] = newSpec
   887  	}
   888  
   889  	return urlSpec
   890  }
   891  
   892  func (a APIDefinitionLoader) compileUnTrackedEndpointPathspathSpec(paths []apidef.TrackEndpointMeta, stat URLStatus) []URLSpec {
   893  	urlSpec := []URLSpec{}
   894  
   895  	for _, stringSpec := range paths {
   896  		newSpec := URLSpec{}
   897  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   898  		// Extend with method actions
   899  		newSpec.DoNotTrackEndpoint = stringSpec
   900  		urlSpec = append(urlSpec, newSpec)
   901  	}
   902  
   903  	return urlSpec
   904  }
   905  
   906  func (a APIDefinitionLoader) compileInternalPathspathSpec(paths []apidef.InternalMeta, stat URLStatus) []URLSpec {
   907  	urlSpec := []URLSpec{}
   908  
   909  	for _, stringSpec := range paths {
   910  		newSpec := URLSpec{}
   911  		a.generateRegex(stringSpec.Path, &newSpec, stat)
   912  		// Extend with method actions
   913  		newSpec.Internal = stringSpec
   914  		urlSpec = append(urlSpec, newSpec)
   915  	}
   916  
   917  	return urlSpec
   918  }
   919  
   920  func (a APIDefinitionLoader) getExtendedPathSpecs(apiVersionDef apidef.VersionInfo, apiSpec *APISpec) ([]URLSpec, bool) {
   921  	// TODO: New compiler here, needs to put data into a different structure
   922  
   923  	ignoredPaths := a.compileExtendedPathSpec(apiVersionDef.IgnoreEndpointCase, apiVersionDef.ExtendedPaths.Ignored, Ignored)
   924  	blackListPaths := a.compileExtendedPathSpec(apiVersionDef.IgnoreEndpointCase, apiVersionDef.ExtendedPaths.BlackList, BlackList)
   925  	whiteListPaths := a.compileExtendedPathSpec(apiVersionDef.IgnoreEndpointCase, apiVersionDef.ExtendedPaths.WhiteList, WhiteList)
   926  	cachedPaths := a.compileCachedPathSpec(apiVersionDef.ExtendedPaths.Cached, apiVersionDef.ExtendedPaths.AdvanceCacheConfig)
   927  	transformPaths := a.compileTransformPathSpec(apiVersionDef.ExtendedPaths.Transform, Transformed)
   928  	transformResponsePaths := a.compileTransformPathSpec(apiVersionDef.ExtendedPaths.TransformResponse, TransformedResponse)
   929  	transformJQPaths := a.compileTransformJQPathSpec(apiVersionDef.ExtendedPaths.TransformJQ, TransformedJQ)
   930  	transformJQResponsePaths := a.compileTransformJQPathSpec(apiVersionDef.ExtendedPaths.TransformJQResponse, TransformedJQResponse)
   931  	headerTransformPaths := a.compileInjectedHeaderSpec(apiVersionDef.ExtendedPaths.TransformHeader, HeaderInjected)
   932  	headerTransformPathsOnResponse := a.compileInjectedHeaderSpec(apiVersionDef.ExtendedPaths.TransformResponseHeader, HeaderInjectedResponse)
   933  	hardTimeouts := a.compileTimeoutPathSpec(apiVersionDef.ExtendedPaths.HardTimeouts, HardTimeout)
   934  	circuitBreakers := a.compileCircuitBreakerPathSpec(apiVersionDef.ExtendedPaths.CircuitBreaker, CircuitBreaker, apiSpec)
   935  	urlRewrites := a.compileURLRewritesPathSpec(apiVersionDef.ExtendedPaths.URLRewrite, URLRewrite)
   936  	virtualPaths := a.compileVirtualPathspathSpec(apiVersionDef.ExtendedPaths.Virtual, VirtualPath, apiSpec)
   937  	requestSizes := a.compileRequestSizePathSpec(apiVersionDef.ExtendedPaths.SizeLimit, RequestSizeLimit)
   938  	methodTransforms := a.compileMethodTransformSpec(apiVersionDef.ExtendedPaths.MethodTransforms, MethodTransformed)
   939  	trackedPaths := a.compileTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.TrackEndpoints, RequestTracked)
   940  	unTrackedPaths := a.compileUnTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.DoNotTrackEndpoints, RequestNotTracked)
   941  	validateJSON := a.compileValidateJSONPathspathSpec(apiVersionDef.ExtendedPaths.ValidateJSON, ValidateJSONRequest)
   942  	internalPaths := a.compileInternalPathspathSpec(apiVersionDef.ExtendedPaths.Internal, Internal)
   943  
   944  	combinedPath := []URLSpec{}
   945  	combinedPath = append(combinedPath, ignoredPaths...)
   946  	combinedPath = append(combinedPath, blackListPaths...)
   947  	combinedPath = append(combinedPath, whiteListPaths...)
   948  	combinedPath = append(combinedPath, cachedPaths...)
   949  	combinedPath = append(combinedPath, transformPaths...)
   950  	combinedPath = append(combinedPath, transformResponsePaths...)
   951  	combinedPath = append(combinedPath, transformJQPaths...)
   952  	combinedPath = append(combinedPath, transformJQResponsePaths...)
   953  	combinedPath = append(combinedPath, headerTransformPaths...)
   954  	combinedPath = append(combinedPath, headerTransformPathsOnResponse...)
   955  	combinedPath = append(combinedPath, hardTimeouts...)
   956  	combinedPath = append(combinedPath, circuitBreakers...)
   957  	combinedPath = append(combinedPath, urlRewrites...)
   958  	combinedPath = append(combinedPath, requestSizes...)
   959  	combinedPath = append(combinedPath, virtualPaths...)
   960  	combinedPath = append(combinedPath, methodTransforms...)
   961  	combinedPath = append(combinedPath, trackedPaths...)
   962  	combinedPath = append(combinedPath, unTrackedPaths...)
   963  	combinedPath = append(combinedPath, validateJSON...)
   964  	combinedPath = append(combinedPath, internalPaths...)
   965  
   966  	return combinedPath, len(whiteListPaths) > 0
   967  }
   968  
   969  func (a *APISpec) Init(authStore, sessionStore, healthStore, orgStore storage.Handler) {
   970  	a.AuthManager.Init(authStore)
   971  	a.SessionManager.Init(sessionStore)
   972  	a.Health.Init(healthStore)
   973  	a.OrgSessionManager.Init(orgStore)
   974  }
   975  
   976  func (a *APISpec) StopSessionManagerPool() {
   977  	a.SessionManager.Stop()
   978  	a.OrgSessionManager.Stop()
   979  }
   980  
   981  func (a *APISpec) getURLStatus(stat URLStatus) RequestStatus {
   982  	switch stat {
   983  	case Ignored:
   984  		return StatusOkAndIgnore
   985  	case BlackList:
   986  		return EndPointNotAllowed
   987  	case WhiteList:
   988  		return StatusOk
   989  	case Cached:
   990  		return StatusCached
   991  	case Transformed:
   992  		return StatusTransform
   993  	case TransformedJQ:
   994  		return StatusTransformJQ
   995  	case HeaderInjected:
   996  		return StatusHeaderInjected
   997  	case HeaderInjectedResponse:
   998  		return StatusHeaderInjectedResponse
   999  	case TransformedResponse:
  1000  		return StatusTransformResponse
  1001  	case TransformedJQResponse:
  1002  		return StatusTransformJQResponse
  1003  	case HardTimeout:
  1004  		return StatusHardTimeout
  1005  	case CircuitBreaker:
  1006  		return StatusCircuitBreaker
  1007  	case URLRewrite:
  1008  		return StatusURLRewrite
  1009  	case VirtualPath:
  1010  		return StatusVirtualPath
  1011  	case RequestSizeLimit:
  1012  		return StatusRequestSizeControlled
  1013  	case MethodTransformed:
  1014  		return StatusMethodTransformed
  1015  	case RequestTracked:
  1016  		return StatusRequestTracked
  1017  	case RequestNotTracked:
  1018  		return StatusRequestNotTracked
  1019  	case ValidateJSONRequest:
  1020  		return StatusValidateJSON
  1021  	case Internal:
  1022  		return StatusInternal
  1023  
  1024  	default:
  1025  		log.Error("URL Status was not one of Ignored, Blacklist or WhiteList! Blocking.")
  1026  		return EndPointNotAllowed
  1027  	}
  1028  }
  1029  
  1030  // URLAllowedAndIgnored checks if a url is allowed and ignored.
  1031  func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, whiteListStatus bool) (RequestStatus, interface{}) {
  1032  	// Check if ignored
  1033  	for i := range rxPaths {
  1034  		if !rxPaths[i].Spec.MatchString(r.URL.Path) {
  1035  			continue
  1036  		}
  1037  
  1038  		if rxPaths[i].MethodActions != nil {
  1039  			// We are using an extended path set, check for the method
  1040  			methodMeta, matchMethodOk := rxPaths[i].MethodActions[r.Method]
  1041  			if !matchMethodOk {
  1042  				continue
  1043  			}
  1044  
  1045  			// Matched the method, check what status it is
  1046  			// TODO: Extend here for additional reply options
  1047  			switch methodMeta.Action {
  1048  			case apidef.NoAction:
  1049  				// NoAction status means we're not treating this request in any special or exceptional way
  1050  				return a.getURLStatus(rxPaths[i].Status), nil
  1051  			case apidef.Reply:
  1052  				return StatusRedirectFlowByReply, &methodMeta
  1053  			default:
  1054  				log.Error("URL Method Action was not set to NoAction, blocking.")
  1055  				return EndPointNotAllowed, nil
  1056  			}
  1057  		}
  1058  
  1059  		if r.Method == rxPaths[i].Internal.Method && rxPaths[i].Status == Internal && !ctxLoopingEnabled(r) {
  1060  			return EndPointNotAllowed, nil
  1061  		}
  1062  
  1063  		if whiteListStatus {
  1064  			// We have a whitelist, nothing gets through unless specifically defined
  1065  			switch rxPaths[i].Status {
  1066  			case WhiteList, BlackList, Ignored:
  1067  			default:
  1068  				if rxPaths[i].Status == Internal && r.Method == rxPaths[i].Internal.Method && ctxLoopingEnabled(r) {
  1069  					return a.getURLStatus(rxPaths[i].Status), nil
  1070  				} else {
  1071  					return EndPointNotAllowed, nil
  1072  				}
  1073  			}
  1074  		}
  1075  
  1076  		if rxPaths[i].TransformAction.Template != nil {
  1077  			return a.getURLStatus(rxPaths[i].Status), &rxPaths[i].TransformAction
  1078  		}
  1079  
  1080  		if rxPaths[i].TransformJQAction.Filter != "" {
  1081  			return a.getURLStatus(rxPaths[i].Status), &rxPaths[i].TransformJQAction
  1082  		}
  1083  
  1084  		// TODO: Fix, Not a great detection method
  1085  		if len(rxPaths[i].InjectHeaders.Path) > 0 {
  1086  			return a.getURLStatus(rxPaths[i].Status), &rxPaths[i].InjectHeaders
  1087  		}
  1088  
  1089  		// Using a legacy path, handle it raw.
  1090  		return a.getURLStatus(rxPaths[i].Status), nil
  1091  	}
  1092  
  1093  	// Nothing matched - should we still let it through?
  1094  	if whiteListStatus {
  1095  		// We have a whitelist, nothing gets through unless specifically defined
  1096  		return EndPointNotAllowed, nil
  1097  	}
  1098  
  1099  	// No whitelist, but also not in any of the other lists, let it through and filter
  1100  	return StatusOk, nil
  1101  }
  1102  
  1103  // CheckSpecMatchesStatus checks if a url spec has a specific status
  1104  func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mode URLStatus) (bool, interface{}) {
  1105  	var matchPath, method string
  1106  
  1107  	//If url-rewrite middleware was used, call response middleware of original path and not of rewritten path
  1108  	// context variable UrlRewritePath is set by rewrite middleware
  1109  	if mode == TransformedJQResponse || mode == HeaderInjectedResponse || mode == TransformedResponse {
  1110  		matchPath = ctxGetUrlRewritePath(r)
  1111  		method = ctxGetRequestMethod(r)
  1112  		if matchPath == "" {
  1113  			matchPath = r.URL.Path
  1114  		}
  1115  	} else {
  1116  		matchPath = r.URL.Path
  1117  		method = r.Method
  1118  	}
  1119  
  1120  	if a.Proxy.ListenPath != "/" {
  1121  		matchPath = strings.TrimPrefix(matchPath, a.Proxy.ListenPath)
  1122  	}
  1123  
  1124  	if !strings.HasPrefix(matchPath, "/") {
  1125  		matchPath = "/" + matchPath
  1126  	}
  1127  
  1128  	// Check if ignored
  1129  	for i := range rxPaths {
  1130  		if mode != rxPaths[i].Status {
  1131  			continue
  1132  		}
  1133  		if !rxPaths[i].Spec.MatchString(matchPath) {
  1134  			continue
  1135  		}
  1136  
  1137  		switch rxPaths[i].Status {
  1138  		case Ignored, BlackList, WhiteList:
  1139  			return true, nil
  1140  		case Cached:
  1141  			if method == rxPaths[i].CacheConfig.Method || (rxPaths[i].CacheConfig.Method == SAFE_METHODS && isSafeMethod(method)) {
  1142  				return true, &rxPaths[i].CacheConfig
  1143  			}
  1144  		case Transformed:
  1145  			if method == rxPaths[i].TransformAction.Method {
  1146  				return true, &rxPaths[i].TransformAction
  1147  			}
  1148  		case TransformedJQ:
  1149  			if method == rxPaths[i].TransformJQAction.Method {
  1150  				return true, &rxPaths[i].TransformJQAction
  1151  			}
  1152  		case HeaderInjected:
  1153  			if method == rxPaths[i].InjectHeaders.Method {
  1154  				return true, &rxPaths[i].InjectHeaders
  1155  			}
  1156  		case HeaderInjectedResponse:
  1157  			if method == rxPaths[i].InjectHeadersResponse.Method {
  1158  				return true, &rxPaths[i].InjectHeadersResponse
  1159  			}
  1160  		case TransformedResponse:
  1161  			if method == rxPaths[i].TransformResponseAction.Method {
  1162  				return true, &rxPaths[i].TransformResponseAction
  1163  			}
  1164  		case TransformedJQResponse:
  1165  			if method == rxPaths[i].TransformJQResponseAction.Method {
  1166  				return true, &rxPaths[i].TransformJQResponseAction
  1167  			}
  1168  		case HardTimeout:
  1169  			if r.Method == rxPaths[i].HardTimeout.Method {
  1170  				return true, &rxPaths[i].HardTimeout.TimeOut
  1171  			}
  1172  		case CircuitBreaker:
  1173  			if method == rxPaths[i].CircuitBreaker.Method {
  1174  				return true, &rxPaths[i].CircuitBreaker
  1175  			}
  1176  		case URLRewrite:
  1177  			if method == rxPaths[i].URLRewrite.Method {
  1178  				return true, rxPaths[i].URLRewrite
  1179  			}
  1180  		case VirtualPath:
  1181  			if method == rxPaths[i].VirtualPathSpec.Method {
  1182  				return true, &rxPaths[i].VirtualPathSpec
  1183  			}
  1184  		case RequestSizeLimit:
  1185  			if method == rxPaths[i].RequestSize.Method {
  1186  				return true, &rxPaths[i].RequestSize
  1187  			}
  1188  		case MethodTransformed:
  1189  			if method == rxPaths[i].MethodTransform.Method {
  1190  				return true, &rxPaths[i].MethodTransform
  1191  			}
  1192  		case RequestTracked:
  1193  			if method == rxPaths[i].TrackEndpoint.Method {
  1194  				return true, &rxPaths[i].TrackEndpoint
  1195  			}
  1196  		case RequestNotTracked:
  1197  			if method == rxPaths[i].DoNotTrackEndpoint.Method {
  1198  				return true, &rxPaths[i].DoNotTrackEndpoint
  1199  			}
  1200  		case ValidateJSONRequest:
  1201  			if method == rxPaths[i].ValidatePathMeta.Method {
  1202  				return true, &rxPaths[i].ValidatePathMeta
  1203  			}
  1204  		case Internal:
  1205  			if method == rxPaths[i].Internal.Method {
  1206  				return true, &rxPaths[i].Internal
  1207  			}
  1208  		}
  1209  	}
  1210  	return false, nil
  1211  }
  1212  
  1213  func (a *APISpec) getVersionFromRequest(r *http.Request) string {
  1214  	if a.VersionData.NotVersioned {
  1215  		return ""
  1216  	}
  1217  
  1218  	switch a.VersionDefinition.Location {
  1219  	case headerLocation:
  1220  		return r.Header.Get(a.VersionDefinition.Key)
  1221  	case urlParamLocation:
  1222  		return r.URL.Query().Get(a.VersionDefinition.Key)
  1223  	case urlLocation:
  1224  		uPath := a.StripListenPath(r, r.URL.Path)
  1225  		uPath = strings.TrimPrefix(uPath, "/"+a.Slug)
  1226  
  1227  		// First non-empty part of the path is the version ID
  1228  		for _, part := range strings.Split(uPath, "/") {
  1229  			if part != "" {
  1230  				return part
  1231  			}
  1232  		}
  1233  	}
  1234  	return ""
  1235  }
  1236  
  1237  // VersionExpired checks if an API version (during a proxied
  1238  // request) is expired. If it isn't and the configured time was valid,
  1239  // it also returns the expiration time.
  1240  func (a *APISpec) VersionExpired(versionDef *apidef.VersionInfo) (bool, *time.Time) {
  1241  	if a.VersionData.NotVersioned {
  1242  		return false, nil
  1243  	}
  1244  
  1245  	// Never expires
  1246  	if versionDef.Expires == "" || versionDef.Expires == "-1" {
  1247  		return false, nil
  1248  	}
  1249  
  1250  	// otherwise use parsed timestamp
  1251  	if versionDef.ExpiresTs.IsZero() {
  1252  		log.Error("Could not parse expiry date for API, disallow")
  1253  		return true, nil
  1254  	}
  1255  
  1256  	// It's in the past, expire
  1257  	// It's in the future, keep going
  1258  	return time.Since(versionDef.ExpiresTs) >= 0, &versionDef.ExpiresTs
  1259  }
  1260  
  1261  // RequestValid will check if an incoming request has valid version
  1262  // data and return a RequestStatus that describes the status of the
  1263  // request
  1264  func (a *APISpec) RequestValid(r *http.Request) (bool, RequestStatus, interface{}) {
  1265  	versionMetaData, versionPaths, whiteListStatus, vstat := a.Version(r)
  1266  
  1267  	// Screwed up version info - fail and pass through
  1268  	if vstat != StatusOk {
  1269  		return false, vstat, nil
  1270  	}
  1271  
  1272  	// Is the API version expired?
  1273  	// TODO: Don't abuse the interface{} return value for both
  1274  	// *apidef.EndpointMethodMeta and *time.Time. Probably need to
  1275  	// redesign or entirely remove RequestValid. See discussion on
  1276  	// https://github.com/TykTechnologies/tyk/pull/776
  1277  	expired, expTime := a.VersionExpired(versionMetaData)
  1278  	if expired {
  1279  		return false, VersionExpired, nil
  1280  	}
  1281  
  1282  	// not expired, let's check path info
  1283  	status, meta := a.URLAllowedAndIgnored(r, versionPaths, whiteListStatus)
  1284  	switch status {
  1285  	case EndPointNotAllowed:
  1286  		return false, status, expTime
  1287  	case StatusRedirectFlowByReply:
  1288  		return true, status, meta
  1289  	case StatusOkAndIgnore, StatusCached, StatusTransform,
  1290  		StatusHeaderInjected, StatusMethodTransformed:
  1291  		return true, status, expTime
  1292  	default:
  1293  		return true, StatusOk, expTime
  1294  	}
  1295  }
  1296  
  1297  // Version attempts to extract the version data from a request, depending on where it is stored in the
  1298  // request (currently only "header" is supported)
  1299  func (a *APISpec) Version(r *http.Request) (*apidef.VersionInfo, []URLSpec, bool, RequestStatus) {
  1300  	var version apidef.VersionInfo
  1301  
  1302  	// try the context first
  1303  	if v := ctxGetVersionInfo(r); v != nil {
  1304  		version = *v
  1305  	} else {
  1306  		// Are we versioned?
  1307  		if a.VersionData.NotVersioned {
  1308  			// Get the first one in the list
  1309  			for _, v := range a.VersionData.Versions {
  1310  				version = v
  1311  				break
  1312  			}
  1313  		} else {
  1314  			// Extract Version Info
  1315  			// First checking for if default version is set
  1316  			vName := a.getVersionFromRequest(r)
  1317  			if vName == "" {
  1318  				if a.VersionData.DefaultVersion == "" {
  1319  					return &version, nil, false, VersionNotFound
  1320  				}
  1321  				vName = a.VersionData.DefaultVersion
  1322  				ctxSetDefaultVersion(r)
  1323  			}
  1324  			// Load Version Data - General
  1325  			var ok bool
  1326  			if version, ok = a.VersionData.Versions[vName]; !ok {
  1327  				return &version, nil, false, VersionDoesNotExist
  1328  			}
  1329  		}
  1330  
  1331  		// cache for the future
  1332  		ctxSetVersionInfo(r, &version)
  1333  	}
  1334  
  1335  	// Load path data and whitelist data for version
  1336  	rxPaths, rxOk := a.RxPaths[version.Name]
  1337  	if !rxOk {
  1338  		log.Error("no RX Paths found for version ", version.Name)
  1339  		return &version, nil, false, VersionDoesNotExist
  1340  	}
  1341  
  1342  	whiteListStatus, wlOk := a.WhiteListEnabled[version.Name]
  1343  	if !wlOk {
  1344  		log.Error("No whitelist data found")
  1345  		return &version, nil, false, VersionWhiteListStatusNotFound
  1346  	}
  1347  
  1348  	return &version, rxPaths, whiteListStatus, StatusOk
  1349  }
  1350  
  1351  func (a *APISpec) StripListenPath(r *http.Request, path string) string {
  1352  	return stripListenPath(a.Proxy.ListenPath, path, mux.Vars(r))
  1353  }
  1354  
  1355  type RoundRobin struct {
  1356  	pos uint32
  1357  }
  1358  
  1359  func (r *RoundRobin) WithLen(len int) int {
  1360  	if len < 1 {
  1361  		return 0
  1362  	}
  1363  	// -1 to start at 0, not 1
  1364  	cur := atomic.AddUint32(&r.pos, 1) - 1
  1365  	return int(cur) % len
  1366  }
  1367  
  1368  var listenPathVarsRE = regexp.MustCompile(`{[^:]+(:[^}]+)?}`)
  1369  
  1370  func stripListenPath(listenPath, path string, muxVars map[string]string) string {
  1371  	if !strings.Contains(listenPath, "{") {
  1372  		return strings.TrimPrefix(path, listenPath)
  1373  	}
  1374  	lp := listenPathVarsRE.ReplaceAllStringFunc(listenPath, func(match string) string {
  1375  		match = strings.TrimLeft(match, "{")
  1376  		match = strings.TrimRight(match, "}")
  1377  		aliasVar := strings.Split(match, ":")[0]
  1378  		return muxVars[aliasVar]
  1379  	})
  1380  	return strings.TrimPrefix(path, lp)
  1381  }