github.com/sequix/cortex@v1.1.6/pkg/chunk/gcp/instrumentation.go (about)

     1  package gcp
     2  
     3  import (
     4  	"context"
     5  	"net/http"
     6  	"strconv"
     7  	"time"
     8  
     9  	otgrpc "github.com/opentracing-contrib/go-grpc"
    10  	opentracing "github.com/opentracing/opentracing-go"
    11  	"github.com/prometheus/client_golang/prometheus"
    12  	"github.com/prometheus/client_golang/prometheus/promauto"
    13  	"google.golang.org/api/option"
    14  	google_http "google.golang.org/api/transport/http"
    15  	"google.golang.org/grpc"
    16  
    17  	"github.com/sequix/cortex/pkg/util/middleware"
    18  )
    19  
    20  var (
    21  	bigtableRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
    22  		Namespace: "cortex",
    23  		Name:      "bigtable_request_duration_seconds",
    24  		Help:      "Time spent doing Bigtable requests.",
    25  
    26  		// Bigtable latency seems to range from a few ms to a few hundred ms and is
    27  		// important.  So use 6 buckets from 1ms to 1s.
    28  		Buckets: prometheus.ExponentialBuckets(0.001, 4, 6),
    29  	}, []string{"operation", "status_code"})
    30  
    31  	gcsRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
    32  		Namespace: "cortex",
    33  		Name:      "gcs_request_duration_seconds",
    34  		Help:      "Time spent doing GCS requests.",
    35  
    36  		// GCS latency seems to range from a few ms to a few secs and is
    37  		// important.  So use 6 buckets from 5ms to 5s.
    38  		Buckets: prometheus.ExponentialBuckets(0.005, 4, 6),
    39  	}, []string{"operation", "status_code"})
    40  )
    41  
    42  func bigtableInstrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterceptor) {
    43  	return []grpc.UnaryClientInterceptor{
    44  			otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
    45  			middleware.PrometheusGRPCUnaryInstrumentation(bigtableRequestDuration),
    46  		},
    47  		[]grpc.StreamClientInterceptor{
    48  			otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()),
    49  			middleware.PrometheusGRPCStreamInstrumentation(bigtableRequestDuration),
    50  		}
    51  }
    52  
    53  func gcsInstrumentation(ctx context.Context, scope string) (option.ClientOption, error) {
    54  	transport, err := google_http.NewTransport(ctx, http.DefaultTransport, option.WithScopes(scope))
    55  	if err != nil {
    56  		return nil, err
    57  	}
    58  	client := &http.Client{
    59  		Transport: instrumentedTransport{
    60  			observer: gcsRequestDuration,
    61  			next:     transport,
    62  		},
    63  	}
    64  	return option.WithHTTPClient(client), nil
    65  }
    66  
    67  func toOptions(opts []grpc.DialOption) []option.ClientOption {
    68  	result := make([]option.ClientOption, 0, len(opts))
    69  	for _, opt := range opts {
    70  		result = append(result, option.WithGRPCDialOption(opt))
    71  	}
    72  	return result
    73  }
    74  
    75  type instrumentedTransport struct {
    76  	observer prometheus.ObserverVec
    77  	next     http.RoundTripper
    78  }
    79  
    80  func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    81  	start := time.Now()
    82  	resp, err := i.next.RoundTrip(req)
    83  	if err == nil {
    84  		i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds())
    85  	}
    86  	return resp, err
    87  }