github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/querier/queryrange/queryrangebase/retry.go (about)

     1  package queryrangebase
     2  
     3  import (
     4  	"context"
     5  
     6  	"github.com/go-kit/log"
     7  	"github.com/go-kit/log/level"
     8  	"github.com/prometheus/client_golang/prometheus"
     9  	"github.com/prometheus/client_golang/prometheus/promauto"
    10  	"github.com/weaveworks/common/httpgrpc"
    11  
    12  	util_log "github.com/grafana/loki/pkg/util/log"
    13  )
    14  
    15  type RetryMiddlewareMetrics struct {
    16  	retriesCount prometheus.Histogram
    17  }
    18  
    19  func NewRetryMiddlewareMetrics(registerer prometheus.Registerer) *RetryMiddlewareMetrics {
    20  	return &RetryMiddlewareMetrics{
    21  		retriesCount: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{
    22  			Namespace: "cortex",
    23  			Name:      "query_frontend_retries",
    24  			Help:      "Number of times a request is retried.",
    25  			Buckets:   []float64{0, 1, 2, 3, 4, 5},
    26  		}),
    27  	}
    28  }
    29  
    30  type retry struct {
    31  	log        log.Logger
    32  	next       Handler
    33  	maxRetries int
    34  
    35  	metrics *RetryMiddlewareMetrics
    36  }
    37  
    38  // NewRetryMiddleware returns a middleware that retries requests if they
    39  // fail with 500 or a non-HTTP error.
    40  func NewRetryMiddleware(log log.Logger, maxRetries int, metrics *RetryMiddlewareMetrics) Middleware {
    41  	if metrics == nil {
    42  		metrics = NewRetryMiddlewareMetrics(nil)
    43  	}
    44  
    45  	return MiddlewareFunc(func(next Handler) Handler {
    46  		return retry{
    47  			log:        log,
    48  			next:       next,
    49  			maxRetries: maxRetries,
    50  			metrics:    metrics,
    51  		}
    52  	})
    53  }
    54  
    55  func (r retry) Do(ctx context.Context, req Request) (Response, error) {
    56  	tries := 0
    57  	defer func() { r.metrics.retriesCount.Observe(float64(tries)) }()
    58  
    59  	var lastErr error
    60  	for ; tries < r.maxRetries; tries++ {
    61  		if ctx.Err() != nil {
    62  			return nil, ctx.Err()
    63  		}
    64  		resp, err := r.next.Do(ctx, req)
    65  		if err == nil {
    66  			return resp, nil
    67  		}
    68  
    69  		// Retry if we get a HTTP 500 or a non-HTTP error.
    70  		httpResp, ok := httpgrpc.HTTPResponseFromError(err)
    71  		if !ok || httpResp.Code/100 == 5 {
    72  			lastErr = err
    73  			level.Error(util_log.WithContext(ctx, r.log)).Log("msg", "error processing request", "try", tries, "err", err)
    74  			continue
    75  		}
    76  
    77  		return nil, err
    78  	}
    79  	return nil, lastErr
    80  }