github.com/Tyktechnologies/tyk@v2.9.5+incompatible/gateway/analytics.go (about)

     1  package gateway
     2  
     3  import (
     4  	"fmt"
     5  	"net"
     6  	"strings"
     7  	"sync"
     8  	"sync/atomic"
     9  	"time"
    10  
    11  	maxminddb "github.com/oschwald/maxminddb-golang"
    12  	msgpack "gopkg.in/vmihailenco/msgpack.v2"
    13  
    14  	"github.com/TykTechnologies/tyk/config"
    15  	"github.com/TykTechnologies/tyk/regexp"
    16  	"github.com/TykTechnologies/tyk/storage"
    17  )
    18  
    19  type NetworkStats struct {
    20  	OpenConnections  int64
    21  	ClosedConnection int64
    22  	BytesIn          int64
    23  	BytesOut         int64
    24  }
    25  
    26  func (n *NetworkStats) Flush() NetworkStats {
    27  	s := NetworkStats{
    28  		OpenConnections:  atomic.LoadInt64(&n.OpenConnections),
    29  		ClosedConnection: atomic.LoadInt64(&n.ClosedConnection),
    30  		BytesIn:          atomic.LoadInt64(&n.BytesIn),
    31  		BytesOut:         atomic.LoadInt64(&n.BytesOut),
    32  	}
    33  	atomic.StoreInt64(&n.OpenConnections, 0)
    34  	atomic.StoreInt64(&n.ClosedConnection, 0)
    35  	atomic.StoreInt64(&n.BytesIn, 0)
    36  	atomic.StoreInt64(&n.BytesOut, 0)
    37  	return s
    38  }
    39  
    40  type Latency struct {
    41  	Total    int64
    42  	Upstream int64
    43  }
    44  
    45  // AnalyticsRecord encodes the details of a request
    46  type AnalyticsRecord struct {
    47  	Method        string
    48  	Host          string
    49  	Path          string // HTTP path, can be overriden by "track path" plugin
    50  	RawPath       string // Original HTTP path
    51  	ContentLength int64
    52  	UserAgent     string
    53  	Day           int
    54  	Month         time.Month
    55  	Year          int
    56  	Hour          int
    57  	ResponseCode  int
    58  	APIKey        string
    59  	TimeStamp     time.Time
    60  	APIVersion    string
    61  	APIName       string
    62  	APIID         string
    63  	OrgID         string
    64  	OauthID       string
    65  	RequestTime   int64
    66  	Latency       Latency
    67  	RawRequest    string // Base64 encoded request data (if detailed recording turned on)
    68  	RawResponse   string // ^ same but for response
    69  	IPAddress     string
    70  	Geo           GeoData
    71  	Network       NetworkStats
    72  	Tags          []string
    73  	Alias         string
    74  	TrackPath     bool
    75  	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
    76  }
    77  
    78  type GeoData struct {
    79  	Country struct {
    80  		ISOCode string `maxminddb:"iso_code"`
    81  	} `maxminddb:"country"`
    82  
    83  	City struct {
    84  		Names map[string]string `maxminddb:"names"`
    85  	} `maxminddb:"city"`
    86  
    87  	Location struct {
    88  		Latitude  float64 `maxminddb:"latitude"`
    89  		Longitude float64 `maxminddb:"longitude"`
    90  		TimeZone  string  `maxminddb:"time_zone"`
    91  	} `maxminddb:"location"`
    92  }
    93  
    94  const analyticsKeyName = "tyk-system-analytics"
    95  
    96  const (
    97  	minRecordsBufferSize             = 1000
    98  	recordsBufferFlushInterval       = 200 * time.Millisecond
    99  	recordsBufferForcedFlushInterval = 1 * time.Second
   100  )
   101  
   102  func (a *AnalyticsRecord) GetGeo(ipStr string) {
   103  	// Not great, tightly coupled
   104  	if analytics.GeoIPDB == nil {
   105  		return
   106  	}
   107  
   108  	record, err := geoIPLookup(ipStr)
   109  	if err != nil {
   110  		log.Error("GeoIP Failure (not recorded): ", err)
   111  		return
   112  	}
   113  	if record == nil {
   114  		return
   115  	}
   116  
   117  	log.Debug("ISO Code: ", record.Country.ISOCode)
   118  	log.Debug("City: ", record.City.Names["en"])
   119  	log.Debug("Lat: ", record.Location.Latitude)
   120  	log.Debug("Lon: ", record.Location.Longitude)
   121  	log.Debug("TZ: ", record.Location.TimeZone)
   122  
   123  	a.Geo = *record
   124  }
   125  
   126  func geoIPLookup(ipStr string) (*GeoData, error) {
   127  	if ipStr == "" {
   128  		return nil, nil
   129  	}
   130  	ip := net.ParseIP(ipStr)
   131  	if ip == nil {
   132  		return nil, fmt.Errorf("invalid IP address %q", ipStr)
   133  	}
   134  	record := new(GeoData)
   135  	if err := analytics.GeoIPDB.Lookup(ip, record); err != nil {
   136  		return nil, fmt.Errorf("geoIPDB lookup of %q failed: %v", ipStr, err)
   137  	}
   138  	return record, nil
   139  }
   140  
   141  func initNormalisationPatterns() (pats config.NormaliseURLPatterns) {
   142  	pats.UUIDs = regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
   143  	pats.IDs = regexp.MustCompile(`\/(\d+)`)
   144  
   145  	for _, pattern := range config.Global().AnalyticsConfig.NormaliseUrls.Custom {
   146  		if patRe, err := regexp.Compile(pattern); err != nil {
   147  			log.Error("failed to compile custom pattern: ", err)
   148  		} else {
   149  			pats.Custom = append(pats.Custom, patRe)
   150  		}
   151  	}
   152  	return
   153  }
   154  
   155  func (a *AnalyticsRecord) NormalisePath(globalConfig *config.Config) {
   156  	if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
   157  		a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
   158  	}
   159  	if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
   160  		a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
   161  	}
   162  	for _, r := range globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom {
   163  		a.Path = r.ReplaceAllString(a.Path, "{var}")
   164  	}
   165  }
   166  
   167  func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) {
   168  	expiry := time.Duration(expiresInSeconds) * time.Second
   169  	if expiresInSeconds == 0 {
   170  		// Expiry is set to 100 years
   171  		expiry = (24 * time.Hour) * (365 * 100)
   172  	}
   173  
   174  	t := time.Now()
   175  	t2 := t.Add(expiry)
   176  	a.ExpireAt = t2
   177  }
   178  
   179  // RedisAnalyticsHandler will record analytics data to a redis back end
   180  // as defined in the Config object
   181  type RedisAnalyticsHandler struct {
   182  	Store            storage.Handler
   183  	Clean            Purger
   184  	GeoIPDB          *maxminddb.Reader
   185  	globalConf       config.Config
   186  	recordsChan      chan *AnalyticsRecord
   187  	workerBufferSize uint64
   188  	shouldStop       uint32
   189  	poolWg           sync.WaitGroup
   190  }
   191  
   192  func (r *RedisAnalyticsHandler) Init(globalConf config.Config) {
   193  	r.globalConf = globalConf
   194  
   195  	if r.globalConf.AnalyticsConfig.EnableGeoIP {
   196  		if db, err := maxminddb.Open(r.globalConf.AnalyticsConfig.GeoIPDBLocation); err != nil {
   197  			log.Error("Failed to init GeoIP Database: ", err)
   198  		} else {
   199  			r.GeoIPDB = db
   200  		}
   201  	}
   202  
   203  	analytics.Store.Connect()
   204  
   205  	ps := r.globalConf.AnalyticsConfig.PoolSize
   206  	if ps == 0 {
   207  		ps = 50
   208  	}
   209  	log.WithField("ps", ps).Debug("Analytics pool workers number")
   210  
   211  	recordsBufferSize := r.globalConf.AnalyticsConfig.RecordsBufferSize
   212  	if recordsBufferSize < minRecordsBufferSize {
   213  		recordsBufferSize = minRecordsBufferSize // force it to this value
   214  	}
   215  	log.WithField("recordsBufferSize", recordsBufferSize).Debug("Analytics total buffer (channel) size")
   216  
   217  	r.workerBufferSize = recordsBufferSize / uint64(ps)
   218  	log.WithField("workerBufferSize", r.workerBufferSize).Debug("Analytics pool worker buffer size")
   219  
   220  	r.recordsChan = make(chan *AnalyticsRecord, recordsBufferSize)
   221  
   222  	// start worker pool
   223  	atomic.SwapUint32(&r.shouldStop, 0)
   224  	for i := 0; i < ps; i++ {
   225  		r.poolWg.Add(1)
   226  		go r.recordWorker()
   227  	}
   228  }
   229  
   230  func (r *RedisAnalyticsHandler) Stop() {
   231  	// flag to stop sending records into channel
   232  	atomic.SwapUint32(&r.shouldStop, 1)
   233  
   234  	// close channel to stop workers
   235  	close(r.recordsChan)
   236  
   237  	// wait for all workers to be done
   238  	r.poolWg.Wait()
   239  }
   240  
   241  // RecordHit will store an AnalyticsRecord in Redis
   242  func (r *RedisAnalyticsHandler) RecordHit(record *AnalyticsRecord) error {
   243  	// check if we should stop sending records 1st
   244  	if atomic.LoadUint32(&r.shouldStop) > 0 {
   245  		return nil
   246  	}
   247  
   248  	// just send record to channel consumed by pool of workers
   249  	// leave all data crunching and Redis I/O work for pool workers
   250  	r.recordsChan <- record
   251  
   252  	return nil
   253  }
   254  
   255  func (r *RedisAnalyticsHandler) recordWorker() {
   256  	defer r.poolWg.Done()
   257  
   258  	// this is buffer to send one pipelined command to redis
   259  	// use r.recordsBufferSize as cap to reduce slice re-allocations
   260  	recordsBuffer := make([]string, 0, r.workerBufferSize)
   261  
   262  	// read records from channel and process
   263  	lastSentTs := time.Now()
   264  	for {
   265  		readyToSend := false
   266  		select {
   267  
   268  		case record, ok := <-r.recordsChan:
   269  			// check if channel was closed and it is time to exit from worker
   270  			if !ok {
   271  				// send what is left in buffer
   272  				r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer)
   273  				return
   274  			}
   275  
   276  			// we have new record - prepare it and add to buffer
   277  
   278  			// If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function)
   279  			record.APIKey = storage.HashKey(record.APIKey)
   280  
   281  			if r.globalConf.SlaveOptions.UseRPC {
   282  				// Extend tag list to include this data so wecan segment by node if necessary
   283  				record.Tags = append(record.Tags, "tyk-hybrid-rpc")
   284  			}
   285  
   286  			if r.globalConf.DBAppConfOptions.NodeIsSegmented {
   287  				// Extend tag list to include this data so we can segment by node if necessary
   288  				record.Tags = append(record.Tags, r.globalConf.DBAppConfOptions.Tags...)
   289  			}
   290  
   291  			// Lets add some metadata
   292  			if record.APIKey != "" {
   293  				record.Tags = append(record.Tags, "key-"+record.APIKey)
   294  			}
   295  
   296  			if record.OrgID != "" {
   297  				record.Tags = append(record.Tags, "org-"+record.OrgID)
   298  			}
   299  
   300  			record.Tags = append(record.Tags, "api-"+record.APIID)
   301  
   302  			// fix paths in record as they might have omitted leading "/"
   303  			if !strings.HasPrefix(record.Path, "/") {
   304  				record.Path = "/" + record.Path
   305  			}
   306  			if !strings.HasPrefix(record.RawPath, "/") {
   307  				record.RawPath = "/" + record.RawPath
   308  			}
   309  
   310  			if encoded, err := msgpack.Marshal(record); err != nil {
   311  				log.WithError(err).Error("Error encoding analytics data")
   312  			} else {
   313  				recordsBuffer = append(recordsBuffer, string(encoded))
   314  			}
   315  
   316  			// identify that buffer is ready to be sent
   317  			readyToSend = uint64(len(recordsBuffer)) == r.workerBufferSize
   318  
   319  		case <-time.After(recordsBufferFlushInterval):
   320  			// nothing was received for that period of time
   321  			// anyways send whatever we have, don't hold data too long in buffer
   322  			readyToSend = true
   323  		}
   324  
   325  		// send data to Redis and reset buffer
   326  		if len(recordsBuffer) > 0 && (readyToSend || time.Since(lastSentTs) >= recordsBufferForcedFlushInterval) {
   327  			r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer)
   328  			recordsBuffer = make([]string, 0, r.workerBufferSize)
   329  			lastSentTs = time.Now()
   330  		}
   331  	}
   332  }
   333  
   334  func DurationToMillisecond(d time.Duration) float64 {
   335  	return float64(d) / 1e6
   336  }