vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletserver/throttle/check.go (about) 1 /* 2 Copyright 2017 GitHub Inc. 3 4 Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE 5 */ 6 7 package throttle 8 9 import ( 10 "context" 11 "fmt" 12 "net/http" 13 "strings" 14 "sync/atomic" 15 "time" 16 17 "vitess.io/vitess/go/stats" 18 "vitess.io/vitess/go/textutil" 19 "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" 20 ) 21 22 const ( 23 // DefaultAppName is the app name used by vitess when app doesn't indicate its name 24 DefaultAppName = "default" 25 vitessAppName = "vitess" 26 27 selfCheckInterval = 250 * time.Millisecond 28 ) 29 30 // CheckFlags provide hints for a check 31 type CheckFlags struct { 32 ReadCheck bool 33 OverrideThreshold float64 34 LowPriority bool 35 OKIfNotExists bool 36 SkipRequestHeartbeats bool 37 } 38 39 // StandardCheckFlags have no special hints 40 var StandardCheckFlags = &CheckFlags{} 41 42 // ThrottlerCheck provides methods for an app checking on metrics 43 type ThrottlerCheck struct { 44 throttler *Throttler 45 } 46 47 // NewThrottlerCheck creates a ThrottlerCheck 48 func NewThrottlerCheck(throttler *Throttler) *ThrottlerCheck { 49 return &ThrottlerCheck{ 50 throttler: throttler, 51 } 52 } 53 54 // checkAppMetricResult allows an app to check on a metric 55 func (check *ThrottlerCheck) checkAppMetricResult(ctx context.Context, appName string, storeType string, storeName string, metricResultFunc base.MetricResultFunc, flags *CheckFlags) (checkResult *CheckResult) { 56 // Handle deprioritized app logic 57 denyApp := false 58 metricName := fmt.Sprintf("%s/%s", storeType, storeName) 59 if flags.LowPriority { 60 if _, exists := check.throttler.nonLowPriorityAppRequestsThrottled.Get(metricName); exists { 61 // a non-deprioritized app, ie a "normal" app, has recently been throttled. 62 // This is now a deprioritized app. Deny access to this request. 63 denyApp = true 64 } 65 } 66 // 67 metricResult, threshold := check.throttler.AppRequestMetricResult(ctx, appName, metricResultFunc, denyApp) 68 if flags.OverrideThreshold > 0 { 69 threshold = flags.OverrideThreshold 70 } 71 value, err := metricResult.Get() 72 if appName == "" { 73 return NewCheckResult(http.StatusExpectationFailed, value, threshold, fmt.Errorf("no app indicated")) 74 } 75 76 var statusCode int 77 78 switch { 79 case err == base.ErrAppDenied: 80 // app specifically not allowed to get metrics 81 statusCode = http.StatusExpectationFailed // 417 82 case err == base.ErrNoSuchMetric: 83 // not collected yet, or metric does not exist 84 statusCode = http.StatusNotFound // 404 85 case err != nil: 86 // any error 87 statusCode = http.StatusInternalServerError // 500 88 case value > threshold: 89 // casual throttling 90 statusCode = http.StatusTooManyRequests // 429 91 err = base.ErrThresholdExceeded 92 93 if !flags.LowPriority && !flags.ReadCheck && appName != vitessAppName { 94 // low priority requests will henceforth be denied 95 go check.throttler.nonLowPriorityAppRequestsThrottled.SetDefault(metricName, true) 96 } 97 default: 98 // all good! 99 statusCode = http.StatusOK // 200 100 } 101 return NewCheckResult(statusCode, value, threshold, err) 102 } 103 104 // Check is the core function that runs when a user wants to check a metric 105 func (check *ThrottlerCheck) Check(ctx context.Context, appName string, storeType string, storeName string, remoteAddr string, flags *CheckFlags) (checkResult *CheckResult) { 106 var metricResultFunc base.MetricResultFunc 107 switch storeType { 108 case "mysql": 109 { 110 metricResultFunc = func() (metricResult base.MetricResult, threshold float64) { 111 return check.throttler.getMySQLClusterMetrics(ctx, storeName) 112 } 113 } 114 } 115 if metricResultFunc == nil { 116 return NoSuchMetricCheckResult 117 } 118 119 checkResult = check.checkAppMetricResult(ctx, appName, storeType, storeName, metricResultFunc, flags) 120 atomic.StoreInt64(&check.throttler.lastCheckTimeNano, time.Now().UnixNano()) 121 122 go func(statusCode int) { 123 stats.GetOrNewCounter("ThrottlerCheckAnyTotal", "total number of checks").Add(1) 124 stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheckAny%s%sTotal", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), "").Add(1) 125 126 if statusCode != http.StatusOK { 127 stats.GetOrNewCounter("ThrottlerCheckAnyError", "total number of failed checks").Add(1) 128 stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheckAny%s%sError", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), "").Add(1) 129 } 130 131 check.throttler.markRecentApp(appName, remoteAddr) 132 }(checkResult.StatusCode) 133 134 return checkResult 135 } 136 137 func (check *ThrottlerCheck) splitMetricTokens(metricName string) (storeType string, storeName string, err error) { 138 metricTokens := strings.Split(metricName, "/") 139 if len(metricTokens) != 2 { 140 return storeType, storeName, base.ErrNoSuchMetric 141 } 142 storeType = metricTokens[0] 143 storeName = metricTokens[1] 144 145 return storeType, storeName, nil 146 } 147 148 // localCheck 149 func (check *ThrottlerCheck) localCheck(ctx context.Context, metricName string) (checkResult *CheckResult) { 150 storeType, storeName, err := check.splitMetricTokens(metricName) 151 if err != nil { 152 return NoSuchMetricCheckResult 153 } 154 checkResult = check.Check(ctx, vitessAppName, storeType, storeName, "local", StandardCheckFlags) 155 156 if checkResult.StatusCode == http.StatusOK { 157 check.throttler.markMetricHealthy(metricName) 158 } 159 if timeSinceHealthy, found := check.throttler.timeSinceMetricHealthy(metricName); found { 160 stats.GetOrNewGauge(fmt.Sprintf("ThrottlerCheck%s%sSecondsSinceHealthy", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), fmt.Sprintf("seconds since last healthy cehck for %s.%s", storeType, storeName)).Set(int64(timeSinceHealthy.Seconds())) 161 } 162 163 return checkResult 164 } 165 166 func (check *ThrottlerCheck) reportAggregated(metricName string, metricResult base.MetricResult) { 167 storeType, storeName, err := check.splitMetricTokens(metricName) 168 if err != nil { 169 return 170 } 171 if value, err := metricResult.Get(); err == nil { 172 stats.GetOrNewGaugeFloat64(fmt.Sprintf("ThrottlerAggregated%s%s", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), fmt.Sprintf("aggregated value for %s.%s", storeType, storeName)).Set(value) 173 } 174 } 175 176 // AggregatedMetrics is a convenience access method into throttler's `aggregatedMetricsSnapshot` 177 func (check *ThrottlerCheck) AggregatedMetrics(ctx context.Context) map[string]base.MetricResult { 178 return check.throttler.aggregatedMetricsSnapshot() 179 } 180 181 // MetricsHealth is a convenience access method into throttler's `metricsHealthSnapshot` 182 func (check *ThrottlerCheck) MetricsHealth() map[string](*base.MetricHealth) { 183 return check.throttler.metricsHealthSnapshot() 184 } 185 186 // SelfChecks runs checks on all known metrics as if we were an app. 187 // This runs asynchronously, continuously, and independently of any user interaction 188 func (check *ThrottlerCheck) SelfChecks(ctx context.Context) { 189 selfCheckTicker := time.NewTicker(selfCheckInterval) 190 go func() { 191 for { 192 select { 193 case <-ctx.Done(): 194 return 195 case <-selfCheckTicker.C: 196 for metricName, metricResult := range check.AggregatedMetrics(ctx) { 197 metricName := metricName 198 metricResult := metricResult 199 go check.localCheck(ctx, metricName) 200 go check.reportAggregated(metricName, metricResult) 201 } 202 } 203 } 204 }() 205 }