github.com/Tyktechnologies/tyk@v2.9.5+incompatible/gateway/analytics.go (about) 1 package gateway 2 3 import ( 4 "fmt" 5 "net" 6 "strings" 7 "sync" 8 "sync/atomic" 9 "time" 10 11 maxminddb "github.com/oschwald/maxminddb-golang" 12 msgpack "gopkg.in/vmihailenco/msgpack.v2" 13 14 "github.com/TykTechnologies/tyk/config" 15 "github.com/TykTechnologies/tyk/regexp" 16 "github.com/TykTechnologies/tyk/storage" 17 ) 18 19 type NetworkStats struct { 20 OpenConnections int64 21 ClosedConnection int64 22 BytesIn int64 23 BytesOut int64 24 } 25 26 func (n *NetworkStats) Flush() NetworkStats { 27 s := NetworkStats{ 28 OpenConnections: atomic.LoadInt64(&n.OpenConnections), 29 ClosedConnection: atomic.LoadInt64(&n.ClosedConnection), 30 BytesIn: atomic.LoadInt64(&n.BytesIn), 31 BytesOut: atomic.LoadInt64(&n.BytesOut), 32 } 33 atomic.StoreInt64(&n.OpenConnections, 0) 34 atomic.StoreInt64(&n.ClosedConnection, 0) 35 atomic.StoreInt64(&n.BytesIn, 0) 36 atomic.StoreInt64(&n.BytesOut, 0) 37 return s 38 } 39 40 type Latency struct { 41 Total int64 42 Upstream int64 43 } 44 45 // AnalyticsRecord encodes the details of a request 46 type AnalyticsRecord struct { 47 Method string 48 Host string 49 Path string // HTTP path, can be overriden by "track path" plugin 50 RawPath string // Original HTTP path 51 ContentLength int64 52 UserAgent string 53 Day int 54 Month time.Month 55 Year int 56 Hour int 57 ResponseCode int 58 APIKey string 59 TimeStamp time.Time 60 APIVersion string 61 APIName string 62 APIID string 63 OrgID string 64 OauthID string 65 RequestTime int64 66 Latency Latency 67 RawRequest string // Base64 encoded request data (if detailed recording turned on) 68 RawResponse string // ^ same but for response 69 IPAddress string 70 Geo GeoData 71 Network NetworkStats 72 Tags []string 73 Alias string 74 TrackPath bool 75 ExpireAt time.Time `bson:"expireAt" json:"expireAt"` 76 } 77 78 type GeoData struct { 79 Country struct { 80 ISOCode string `maxminddb:"iso_code"` 81 } `maxminddb:"country"` 82 83 City struct { 84 Names map[string]string `maxminddb:"names"` 85 } `maxminddb:"city"` 86 87 Location struct { 88 Latitude float64 `maxminddb:"latitude"` 89 Longitude float64 `maxminddb:"longitude"` 90 TimeZone string `maxminddb:"time_zone"` 91 } `maxminddb:"location"` 92 } 93 94 const analyticsKeyName = "tyk-system-analytics" 95 96 const ( 97 minRecordsBufferSize = 1000 98 recordsBufferFlushInterval = 200 * time.Millisecond 99 recordsBufferForcedFlushInterval = 1 * time.Second 100 ) 101 102 func (a *AnalyticsRecord) GetGeo(ipStr string) { 103 // Not great, tightly coupled 104 if analytics.GeoIPDB == nil { 105 return 106 } 107 108 record, err := geoIPLookup(ipStr) 109 if err != nil { 110 log.Error("GeoIP Failure (not recorded): ", err) 111 return 112 } 113 if record == nil { 114 return 115 } 116 117 log.Debug("ISO Code: ", record.Country.ISOCode) 118 log.Debug("City: ", record.City.Names["en"]) 119 log.Debug("Lat: ", record.Location.Latitude) 120 log.Debug("Lon: ", record.Location.Longitude) 121 log.Debug("TZ: ", record.Location.TimeZone) 122 123 a.Geo = *record 124 } 125 126 func geoIPLookup(ipStr string) (*GeoData, error) { 127 if ipStr == "" { 128 return nil, nil 129 } 130 ip := net.ParseIP(ipStr) 131 if ip == nil { 132 return nil, fmt.Errorf("invalid IP address %q", ipStr) 133 } 134 record := new(GeoData) 135 if err := analytics.GeoIPDB.Lookup(ip, record); err != nil { 136 return nil, fmt.Errorf("geoIPDB lookup of %q failed: %v", ipStr, err) 137 } 138 return record, nil 139 } 140 141 func initNormalisationPatterns() (pats config.NormaliseURLPatterns) { 142 pats.UUIDs = regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`) 143 pats.IDs = regexp.MustCompile(`\/(\d+)`) 144 145 for _, pattern := range config.Global().AnalyticsConfig.NormaliseUrls.Custom { 146 if patRe, err := regexp.Compile(pattern); err != nil { 147 log.Error("failed to compile custom pattern: ", err) 148 } else { 149 pats.Custom = append(pats.Custom, patRe) 150 } 151 } 152 return 153 } 154 155 func (a *AnalyticsRecord) NormalisePath(globalConfig *config.Config) { 156 if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs { 157 a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}") 158 } 159 if globalConfig.AnalyticsConfig.NormaliseUrls.NormaliseNumbers { 160 a.Path = globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}") 161 } 162 for _, r := range globalConfig.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom { 163 a.Path = r.ReplaceAllString(a.Path, "{var}") 164 } 165 } 166 167 func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) { 168 expiry := time.Duration(expiresInSeconds) * time.Second 169 if expiresInSeconds == 0 { 170 // Expiry is set to 100 years 171 expiry = (24 * time.Hour) * (365 * 100) 172 } 173 174 t := time.Now() 175 t2 := t.Add(expiry) 176 a.ExpireAt = t2 177 } 178 179 // RedisAnalyticsHandler will record analytics data to a redis back end 180 // as defined in the Config object 181 type RedisAnalyticsHandler struct { 182 Store storage.Handler 183 Clean Purger 184 GeoIPDB *maxminddb.Reader 185 globalConf config.Config 186 recordsChan chan *AnalyticsRecord 187 workerBufferSize uint64 188 shouldStop uint32 189 poolWg sync.WaitGroup 190 } 191 192 func (r *RedisAnalyticsHandler) Init(globalConf config.Config) { 193 r.globalConf = globalConf 194 195 if r.globalConf.AnalyticsConfig.EnableGeoIP { 196 if db, err := maxminddb.Open(r.globalConf.AnalyticsConfig.GeoIPDBLocation); err != nil { 197 log.Error("Failed to init GeoIP Database: ", err) 198 } else { 199 r.GeoIPDB = db 200 } 201 } 202 203 analytics.Store.Connect() 204 205 ps := r.globalConf.AnalyticsConfig.PoolSize 206 if ps == 0 { 207 ps = 50 208 } 209 log.WithField("ps", ps).Debug("Analytics pool workers number") 210 211 recordsBufferSize := r.globalConf.AnalyticsConfig.RecordsBufferSize 212 if recordsBufferSize < minRecordsBufferSize { 213 recordsBufferSize = minRecordsBufferSize // force it to this value 214 } 215 log.WithField("recordsBufferSize", recordsBufferSize).Debug("Analytics total buffer (channel) size") 216 217 r.workerBufferSize = recordsBufferSize / uint64(ps) 218 log.WithField("workerBufferSize", r.workerBufferSize).Debug("Analytics pool worker buffer size") 219 220 r.recordsChan = make(chan *AnalyticsRecord, recordsBufferSize) 221 222 // start worker pool 223 atomic.SwapUint32(&r.shouldStop, 0) 224 for i := 0; i < ps; i++ { 225 r.poolWg.Add(1) 226 go r.recordWorker() 227 } 228 } 229 230 func (r *RedisAnalyticsHandler) Stop() { 231 // flag to stop sending records into channel 232 atomic.SwapUint32(&r.shouldStop, 1) 233 234 // close channel to stop workers 235 close(r.recordsChan) 236 237 // wait for all workers to be done 238 r.poolWg.Wait() 239 } 240 241 // RecordHit will store an AnalyticsRecord in Redis 242 func (r *RedisAnalyticsHandler) RecordHit(record *AnalyticsRecord) error { 243 // check if we should stop sending records 1st 244 if atomic.LoadUint32(&r.shouldStop) > 0 { 245 return nil 246 } 247 248 // just send record to channel consumed by pool of workers 249 // leave all data crunching and Redis I/O work for pool workers 250 r.recordsChan <- record 251 252 return nil 253 } 254 255 func (r *RedisAnalyticsHandler) recordWorker() { 256 defer r.poolWg.Done() 257 258 // this is buffer to send one pipelined command to redis 259 // use r.recordsBufferSize as cap to reduce slice re-allocations 260 recordsBuffer := make([]string, 0, r.workerBufferSize) 261 262 // read records from channel and process 263 lastSentTs := time.Now() 264 for { 265 readyToSend := false 266 select { 267 268 case record, ok := <-r.recordsChan: 269 // check if channel was closed and it is time to exit from worker 270 if !ok { 271 // send what is left in buffer 272 r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer) 273 return 274 } 275 276 // we have new record - prepare it and add to buffer 277 278 // If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function) 279 record.APIKey = storage.HashKey(record.APIKey) 280 281 if r.globalConf.SlaveOptions.UseRPC { 282 // Extend tag list to include this data so wecan segment by node if necessary 283 record.Tags = append(record.Tags, "tyk-hybrid-rpc") 284 } 285 286 if r.globalConf.DBAppConfOptions.NodeIsSegmented { 287 // Extend tag list to include this data so we can segment by node if necessary 288 record.Tags = append(record.Tags, r.globalConf.DBAppConfOptions.Tags...) 289 } 290 291 // Lets add some metadata 292 if record.APIKey != "" { 293 record.Tags = append(record.Tags, "key-"+record.APIKey) 294 } 295 296 if record.OrgID != "" { 297 record.Tags = append(record.Tags, "org-"+record.OrgID) 298 } 299 300 record.Tags = append(record.Tags, "api-"+record.APIID) 301 302 // fix paths in record as they might have omitted leading "/" 303 if !strings.HasPrefix(record.Path, "/") { 304 record.Path = "/" + record.Path 305 } 306 if !strings.HasPrefix(record.RawPath, "/") { 307 record.RawPath = "/" + record.RawPath 308 } 309 310 if encoded, err := msgpack.Marshal(record); err != nil { 311 log.WithError(err).Error("Error encoding analytics data") 312 } else { 313 recordsBuffer = append(recordsBuffer, string(encoded)) 314 } 315 316 // identify that buffer is ready to be sent 317 readyToSend = uint64(len(recordsBuffer)) == r.workerBufferSize 318 319 case <-time.After(recordsBufferFlushInterval): 320 // nothing was received for that period of time 321 // anyways send whatever we have, don't hold data too long in buffer 322 readyToSend = true 323 } 324 325 // send data to Redis and reset buffer 326 if len(recordsBuffer) > 0 && (readyToSend || time.Since(lastSentTs) >= recordsBufferForcedFlushInterval) { 327 r.Store.AppendToSetPipelined(analyticsKeyName, recordsBuffer) 328 recordsBuffer = make([]string, 0, r.workerBufferSize) 329 lastSentTs = time.Now() 330 } 331 } 332 } 333 334 func DurationToMillisecond(d time.Duration) float64 { 335 return float64(d) / 1e6 336 }