vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletserver/throttle/check.go (about)

     1  /*
     2   Copyright 2017 GitHub Inc.
     3  
     4   Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE
     5  */
     6  
     7  package throttle
     8  
     9  import (
    10  	"context"
    11  	"fmt"
    12  	"net/http"
    13  	"strings"
    14  	"sync/atomic"
    15  	"time"
    16  
    17  	"vitess.io/vitess/go/stats"
    18  	"vitess.io/vitess/go/textutil"
    19  	"vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base"
    20  )
    21  
    22  const (
    23  	// DefaultAppName is the app name used by vitess when app doesn't indicate its name
    24  	DefaultAppName = "default"
    25  	vitessAppName  = "vitess"
    26  
    27  	selfCheckInterval = 250 * time.Millisecond
    28  )
    29  
    30  // CheckFlags provide hints for a check
    31  type CheckFlags struct {
    32  	ReadCheck             bool
    33  	OverrideThreshold     float64
    34  	LowPriority           bool
    35  	OKIfNotExists         bool
    36  	SkipRequestHeartbeats bool
    37  }
    38  
    39  // StandardCheckFlags have no special hints
    40  var StandardCheckFlags = &CheckFlags{}
    41  
    42  // ThrottlerCheck provides methods for an app checking on metrics
    43  type ThrottlerCheck struct {
    44  	throttler *Throttler
    45  }
    46  
    47  // NewThrottlerCheck creates a ThrottlerCheck
    48  func NewThrottlerCheck(throttler *Throttler) *ThrottlerCheck {
    49  	return &ThrottlerCheck{
    50  		throttler: throttler,
    51  	}
    52  }
    53  
    54  // checkAppMetricResult allows an app to check on a metric
    55  func (check *ThrottlerCheck) checkAppMetricResult(ctx context.Context, appName string, storeType string, storeName string, metricResultFunc base.MetricResultFunc, flags *CheckFlags) (checkResult *CheckResult) {
    56  	// Handle deprioritized app logic
    57  	denyApp := false
    58  	metricName := fmt.Sprintf("%s/%s", storeType, storeName)
    59  	if flags.LowPriority {
    60  		if _, exists := check.throttler.nonLowPriorityAppRequestsThrottled.Get(metricName); exists {
    61  			// a non-deprioritized app, ie a "normal" app, has recently been throttled.
    62  			// This is now a deprioritized app. Deny access to this request.
    63  			denyApp = true
    64  		}
    65  	}
    66  	//
    67  	metricResult, threshold := check.throttler.AppRequestMetricResult(ctx, appName, metricResultFunc, denyApp)
    68  	if flags.OverrideThreshold > 0 {
    69  		threshold = flags.OverrideThreshold
    70  	}
    71  	value, err := metricResult.Get()
    72  	if appName == "" {
    73  		return NewCheckResult(http.StatusExpectationFailed, value, threshold, fmt.Errorf("no app indicated"))
    74  	}
    75  
    76  	var statusCode int
    77  
    78  	switch {
    79  	case err == base.ErrAppDenied:
    80  		// app specifically not allowed to get metrics
    81  		statusCode = http.StatusExpectationFailed // 417
    82  	case err == base.ErrNoSuchMetric:
    83  		// not collected yet, or metric does not exist
    84  		statusCode = http.StatusNotFound // 404
    85  	case err != nil:
    86  		// any error
    87  		statusCode = http.StatusInternalServerError // 500
    88  	case value > threshold:
    89  		// casual throttling
    90  		statusCode = http.StatusTooManyRequests // 429
    91  		err = base.ErrThresholdExceeded
    92  
    93  		if !flags.LowPriority && !flags.ReadCheck && appName != vitessAppName {
    94  			// low priority requests will henceforth be denied
    95  			go check.throttler.nonLowPriorityAppRequestsThrottled.SetDefault(metricName, true)
    96  		}
    97  	default:
    98  		// all good!
    99  		statusCode = http.StatusOK // 200
   100  	}
   101  	return NewCheckResult(statusCode, value, threshold, err)
   102  }
   103  
   104  // Check is the core function that runs when a user wants to check a metric
   105  func (check *ThrottlerCheck) Check(ctx context.Context, appName string, storeType string, storeName string, remoteAddr string, flags *CheckFlags) (checkResult *CheckResult) {
   106  	var metricResultFunc base.MetricResultFunc
   107  	switch storeType {
   108  	case "mysql":
   109  		{
   110  			metricResultFunc = func() (metricResult base.MetricResult, threshold float64) {
   111  				return check.throttler.getMySQLClusterMetrics(ctx, storeName)
   112  			}
   113  		}
   114  	}
   115  	if metricResultFunc == nil {
   116  		return NoSuchMetricCheckResult
   117  	}
   118  
   119  	checkResult = check.checkAppMetricResult(ctx, appName, storeType, storeName, metricResultFunc, flags)
   120  	atomic.StoreInt64(&check.throttler.lastCheckTimeNano, time.Now().UnixNano())
   121  
   122  	go func(statusCode int) {
   123  		stats.GetOrNewCounter("ThrottlerCheckAnyTotal", "total number of checks").Add(1)
   124  		stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheckAny%s%sTotal", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), "").Add(1)
   125  
   126  		if statusCode != http.StatusOK {
   127  			stats.GetOrNewCounter("ThrottlerCheckAnyError", "total number of failed checks").Add(1)
   128  			stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheckAny%s%sError", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), "").Add(1)
   129  		}
   130  
   131  		check.throttler.markRecentApp(appName, remoteAddr)
   132  	}(checkResult.StatusCode)
   133  
   134  	return checkResult
   135  }
   136  
   137  func (check *ThrottlerCheck) splitMetricTokens(metricName string) (storeType string, storeName string, err error) {
   138  	metricTokens := strings.Split(metricName, "/")
   139  	if len(metricTokens) != 2 {
   140  		return storeType, storeName, base.ErrNoSuchMetric
   141  	}
   142  	storeType = metricTokens[0]
   143  	storeName = metricTokens[1]
   144  
   145  	return storeType, storeName, nil
   146  }
   147  
   148  // localCheck
   149  func (check *ThrottlerCheck) localCheck(ctx context.Context, metricName string) (checkResult *CheckResult) {
   150  	storeType, storeName, err := check.splitMetricTokens(metricName)
   151  	if err != nil {
   152  		return NoSuchMetricCheckResult
   153  	}
   154  	checkResult = check.Check(ctx, vitessAppName, storeType, storeName, "local", StandardCheckFlags)
   155  
   156  	if checkResult.StatusCode == http.StatusOK {
   157  		check.throttler.markMetricHealthy(metricName)
   158  	}
   159  	if timeSinceHealthy, found := check.throttler.timeSinceMetricHealthy(metricName); found {
   160  		stats.GetOrNewGauge(fmt.Sprintf("ThrottlerCheck%s%sSecondsSinceHealthy", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), fmt.Sprintf("seconds since last healthy cehck for %s.%s", storeType, storeName)).Set(int64(timeSinceHealthy.Seconds()))
   161  	}
   162  
   163  	return checkResult
   164  }
   165  
   166  func (check *ThrottlerCheck) reportAggregated(metricName string, metricResult base.MetricResult) {
   167  	storeType, storeName, err := check.splitMetricTokens(metricName)
   168  	if err != nil {
   169  		return
   170  	}
   171  	if value, err := metricResult.Get(); err == nil {
   172  		stats.GetOrNewGaugeFloat64(fmt.Sprintf("ThrottlerAggregated%s%s", textutil.SingleWordCamel(storeType), textutil.SingleWordCamel(storeName)), fmt.Sprintf("aggregated value for %s.%s", storeType, storeName)).Set(value)
   173  	}
   174  }
   175  
   176  // AggregatedMetrics is a convenience access method into throttler's `aggregatedMetricsSnapshot`
   177  func (check *ThrottlerCheck) AggregatedMetrics(ctx context.Context) map[string]base.MetricResult {
   178  	return check.throttler.aggregatedMetricsSnapshot()
   179  }
   180  
   181  // MetricsHealth is a convenience access method into throttler's `metricsHealthSnapshot`
   182  func (check *ThrottlerCheck) MetricsHealth() map[string](*base.MetricHealth) {
   183  	return check.throttler.metricsHealthSnapshot()
   184  }
   185  
   186  // SelfChecks runs checks on all known metrics as if we were an app.
   187  // This runs asynchronously, continuously, and independently of any user interaction
   188  func (check *ThrottlerCheck) SelfChecks(ctx context.Context) {
   189  	selfCheckTicker := time.NewTicker(selfCheckInterval)
   190  	go func() {
   191  		for {
   192  			select {
   193  			case <-ctx.Done():
   194  				return
   195  			case <-selfCheckTicker.C:
   196  				for metricName, metricResult := range check.AggregatedMetrics(ctx) {
   197  					metricName := metricName
   198  					metricResult := metricResult
   199  					go check.localCheck(ctx, metricName)
   200  					go check.reportAggregated(metricName, metricResult)
   201  				}
   202  			}
   203  		}
   204  	}()
   205  }