github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/query/api/v1/handler/prometheus/remote/write.go (about)

     1  // Copyright (c) 2018 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package remote
    22  
    23  import (
    24  	"bytes"
    25  	"context"
    26  	"encoding/json"
    27  	"errors"
    28  	"fmt"
    29  	"io/ioutil"
    30  	"net/http"
    31  	"sort"
    32  	"strings"
    33  	"sync/atomic"
    34  	"time"
    35  
    36  	"github.com/m3db/m3/src/cmd/services/m3coordinator/ingest"
    37  	"github.com/m3db/m3/src/dbnode/client"
    38  	"github.com/m3db/m3/src/metrics/policy"
    39  	"github.com/m3db/m3/src/query/api/v1/handler/prometheus"
    40  	"github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions"
    41  	"github.com/m3db/m3/src/query/api/v1/options"
    42  	"github.com/m3db/m3/src/query/api/v1/route"
    43  	"github.com/m3db/m3/src/query/generated/proto/prompb"
    44  	"github.com/m3db/m3/src/query/models"
    45  	"github.com/m3db/m3/src/query/storage"
    46  	"github.com/m3db/m3/src/query/storage/m3/storagemetadata"
    47  	"github.com/m3db/m3/src/query/ts"
    48  	"github.com/m3db/m3/src/query/util/logging"
    49  	"github.com/m3db/m3/src/x/clock"
    50  	xerrors "github.com/m3db/m3/src/x/errors"
    51  	"github.com/m3db/m3/src/x/headers"
    52  	"github.com/m3db/m3/src/x/instrument"
    53  	xhttp "github.com/m3db/m3/src/x/net/http"
    54  	"github.com/m3db/m3/src/x/retry"
    55  	xsync "github.com/m3db/m3/src/x/sync"
    56  	xtime "github.com/m3db/m3/src/x/time"
    57  
    58  	"github.com/cespare/xxhash/v2"
    59  	"github.com/golang/protobuf/proto"
    60  	"github.com/golang/snappy"
    61  	murmur3 "github.com/m3db/stackmurmur3/v2"
    62  	"github.com/uber-go/tally"
    63  	"go.uber.org/zap"
    64  )
    65  
    66  const (
    67  	// PromWriteURL is the url for the prom write handler
    68  	PromWriteURL = route.Prefix + "/prom/remote/write"
    69  
    70  	// PromWriteHTTPMethod is the HTTP method used with this resource.
    71  	PromWriteHTTPMethod = http.MethodPost
    72  
    73  	// emptyStoragePolicyVar for code readability.
    74  	emptyStoragePolicyVar = ""
    75  
    76  	// defaultForwardingTimeout is the default forwarding timeout.
    77  	defaultForwardingTimeout = 15 * time.Second
    78  
    79  	// maxLiteralIsTooLongLogCount is the number of times the time series labels should be logged
    80  	// upon "literal is too long" error.
    81  	maxLiteralIsTooLongLogCount = 10
    82  	// literalPrefixLength is the length of the label literal prefix that is logged upon
    83  	// "literal is too long" error.
    84  	literalPrefixLength = 100
    85  )
    86  
    87  var (
    88  	errNoDownsamplerAndWriter       = errors.New("no downsampler and writer set")
    89  	errNoTagOptions                 = errors.New("no tag options set")
    90  	errNoNowFn                      = errors.New("no now fn set")
    91  	errUnaggregatedStoragePolicySet = errors.New("storage policy should not be set for unaggregated metrics")
    92  
    93  	defaultForwardingRetryForever = false
    94  	defaultForwardingRetryJitter  = true
    95  	defaultForwardRetryConfig     = retry.Configuration{
    96  		InitialBackoff: time.Second * 2,
    97  		BackoffFactor:  2,
    98  		MaxRetries:     1,
    99  		Forever:        &defaultForwardingRetryForever,
   100  		Jitter:         &defaultForwardingRetryJitter,
   101  	}
   102  
   103  	defaultValue = ingest.IterValue{
   104  		Tags:       models.EmptyTags(),
   105  		Attributes: ts.DefaultSeriesAttributes(),
   106  		Metadata:   ts.Metadata{},
   107  	}
   108  
   109  	headerToMetricType = map[string]prompb.MetricType{
   110  		"counter":         prompb.MetricType_COUNTER,
   111  		"gauge":           prompb.MetricType_GAUGE,
   112  		"gauge_histogram": prompb.MetricType_GAUGE_HISTOGRAM,
   113  		"histogram":       prompb.MetricType_HISTOGRAM,
   114  		"info":            prompb.MetricType_INFO,
   115  		"stateset":        prompb.MetricType_STATESET,
   116  		"summary":         prompb.MetricType_SUMMARY,
   117  	}
   118  )
   119  
   120  // PromWriteHandler represents a handler for prometheus write endpoint.
   121  type PromWriteHandler struct {
   122  	downsamplerAndWriter   ingest.DownsamplerAndWriter
   123  	tagOptions             models.TagOptions
   124  	storeMetricsType       bool
   125  	forwarding             handleroptions.PromWriteHandlerForwardingOptions
   126  	forwardTimeout         time.Duration
   127  	forwardHTTPClient      *http.Client
   128  	forwardingBoundWorkers xsync.WorkerPool
   129  	forwardContext         context.Context
   130  	forwardRetrier         retry.Retrier
   131  	nowFn                  clock.NowFn
   132  	instrumentOpts         instrument.Options
   133  	metrics                promWriteMetrics
   134  
   135  	// Counting the number of times of "literal is too long" error for log sampling purposes.
   136  	numLiteralIsTooLong uint32
   137  }
   138  
   139  // NewPromWriteHandler returns a new instance of handler.
   140  func NewPromWriteHandler(options options.HandlerOptions) (http.Handler, error) {
   141  	var (
   142  		downsamplerAndWriter = options.DownsamplerAndWriter()
   143  		tagOptions           = options.TagOptions()
   144  		nowFn                = options.NowFn()
   145  		forwarding           = options.Config().WriteForwarding.PromRemoteWrite
   146  		instrumentOpts       = options.InstrumentOpts()
   147  	)
   148  
   149  	if downsamplerAndWriter == nil {
   150  		return nil, errNoDownsamplerAndWriter
   151  	}
   152  
   153  	if tagOptions == nil {
   154  		return nil, errNoTagOptions
   155  	}
   156  
   157  	if nowFn == nil {
   158  		return nil, errNoNowFn
   159  	}
   160  
   161  	scope := options.InstrumentOpts().
   162  		MetricsScope().
   163  		Tagged(map[string]string{"handler": "remote-write"})
   164  	metrics, err := newPromWriteMetrics(scope)
   165  	if err != nil {
   166  		return nil, err
   167  	}
   168  
   169  	// Only use a forwarding worker pool if concurrency is bound, otherwise
   170  	// if unlimited we just spin up a goroutine for each incoming write.
   171  	var forwardingBoundWorkers xsync.WorkerPool
   172  	if v := forwarding.MaxConcurrency; v > 0 {
   173  		forwardingBoundWorkers = xsync.NewWorkerPool(v)
   174  		forwardingBoundWorkers.Init()
   175  	}
   176  
   177  	forwardTimeout := defaultForwardingTimeout
   178  	if v := forwarding.Timeout; v > 0 {
   179  		forwardTimeout = v
   180  	}
   181  
   182  	forwardHTTPOpts := xhttp.DefaultHTTPClientOptions()
   183  	forwardHTTPOpts.DisableCompression = true // Already snappy compressed.
   184  	forwardHTTPOpts.RequestTimeout = forwardTimeout
   185  
   186  	forwardRetryConfig := defaultForwardRetryConfig
   187  	if forwarding.Retry != nil {
   188  		forwardRetryConfig = *forwarding.Retry
   189  	}
   190  	forwardRetryOpts := forwardRetryConfig.NewOptions(
   191  		scope.SubScope("forwarding-retry"),
   192  	)
   193  
   194  	return &PromWriteHandler{
   195  		downsamplerAndWriter:   downsamplerAndWriter,
   196  		tagOptions:             tagOptions,
   197  		storeMetricsType:       options.StoreMetricsType(),
   198  		forwarding:             forwarding,
   199  		forwardTimeout:         forwardTimeout,
   200  		forwardHTTPClient:      xhttp.NewHTTPClient(forwardHTTPOpts),
   201  		forwardingBoundWorkers: forwardingBoundWorkers,
   202  		forwardContext:         context.Background(),
   203  		forwardRetrier:         retry.NewRetrier(forwardRetryOpts),
   204  		nowFn:                  nowFn,
   205  		metrics:                metrics,
   206  		instrumentOpts:         instrumentOpts,
   207  	}, nil
   208  }
   209  
   210  type promWriteMetrics struct {
   211  	writeSuccess             tally.Counter
   212  	writeErrorsServer        tally.Counter
   213  	writeErrorsClient        tally.Counter
   214  	writeBatchLatency        tally.Histogram
   215  	writeBatchLatencyBuckets tally.DurationBuckets
   216  	ingestLatency            tally.Histogram
   217  	ingestLatencyBuckets     tally.DurationBuckets
   218  	forwardSuccess           tally.Counter
   219  	forwardErrors            tally.Counter
   220  	forwardDropped           tally.Counter
   221  	forwardLatency           tally.Histogram
   222  	forwardShadowKeep        tally.Counter
   223  	forwardShadowDrop        tally.Counter
   224  }
   225  
   226  func (m *promWriteMetrics) incError(err error) {
   227  	if xhttp.IsClientError(err) {
   228  		m.writeErrorsClient.Inc(1)
   229  	} else {
   230  		m.writeErrorsServer.Inc(1)
   231  	}
   232  }
   233  
   234  func newPromWriteMetrics(scope tally.Scope) (promWriteMetrics, error) {
   235  	buckets, err := ingest.NewLatencyBuckets()
   236  	if err != nil {
   237  		return promWriteMetrics{}, err
   238  	}
   239  	return promWriteMetrics{
   240  		writeSuccess:             scope.SubScope("write").Counter("success"),
   241  		writeErrorsServer:        scope.SubScope("write").Tagged(map[string]string{"code": "5XX"}).Counter("errors"),
   242  		writeErrorsClient:        scope.SubScope("write").Tagged(map[string]string{"code": "4XX"}).Counter("errors"),
   243  		writeBatchLatency:        scope.SubScope("write").Histogram("batch-latency", buckets.WriteLatencyBuckets),
   244  		writeBatchLatencyBuckets: buckets.WriteLatencyBuckets,
   245  		ingestLatency:            scope.SubScope("ingest").Histogram("latency", buckets.IngestLatencyBuckets),
   246  		ingestLatencyBuckets:     buckets.IngestLatencyBuckets,
   247  		forwardSuccess:           scope.SubScope("forward").Counter("success"),
   248  		forwardErrors:            scope.SubScope("forward").Counter("errors"),
   249  		forwardDropped:           scope.SubScope("forward").Counter("dropped"),
   250  		forwardLatency:           scope.SubScope("forward").Histogram("latency", buckets.WriteLatencyBuckets),
   251  		forwardShadowKeep:        scope.SubScope("forward").SubScope("shadow").Counter("keep"),
   252  		forwardShadowDrop:        scope.SubScope("forward").SubScope("shadow").Counter("drop"),
   253  	}, nil
   254  }
   255  
   256  func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
   257  	batchRequestStopwatch := h.metrics.writeBatchLatency.Start()
   258  	defer batchRequestStopwatch.Stop()
   259  
   260  	checkedReq, err := h.checkedParseRequest(r)
   261  	if err != nil {
   262  		h.metrics.incError(err)
   263  		xhttp.WriteError(w, err)
   264  		return
   265  	}
   266  
   267  	var (
   268  		req  = checkedReq.Request
   269  		opts = checkedReq.Options
   270  	)
   271  	// Begin async forwarding.
   272  	// NB(r): Be careful about not returning buffers to pool
   273  	// if the request bodies ever get pooled until after
   274  	// forwarding completes.
   275  	if targets := h.forwarding.Targets; len(targets) > 0 {
   276  		for _, target := range targets {
   277  			target := target // Capture for lambda.
   278  			forward := func() {
   279  				now := h.nowFn()
   280  
   281  				var (
   282  					attempt = func() error {
   283  						// Consider propagating baggage without tying
   284  						// context to request context in future.
   285  						ctx, cancel := context.WithTimeout(h.forwardContext, h.forwardTimeout)
   286  						defer cancel()
   287  						return h.forward(ctx, checkedReq, r.Header, target)
   288  					}
   289  					err error
   290  				)
   291  				if target.NoRetry {
   292  					err = attempt()
   293  				} else {
   294  					err = h.forwardRetrier.Attempt(attempt)
   295  				}
   296  
   297  				// Record forward ingestion delay.
   298  				// NB: this includes any time for retries.
   299  				for _, series := range req.Timeseries {
   300  					for _, sample := range series.Samples {
   301  						age := now.Sub(storage.PromTimestampToTime(sample.Timestamp))
   302  						h.metrics.forwardLatency.RecordDuration(age)
   303  					}
   304  				}
   305  
   306  				if err != nil {
   307  					h.metrics.forwardErrors.Inc(1)
   308  					logger := logging.WithContext(h.forwardContext, h.instrumentOpts)
   309  					logger.Error("forward error", zap.Error(err))
   310  					return
   311  				}
   312  
   313  				h.metrics.forwardSuccess.Inc(1)
   314  			}
   315  
   316  			spawned := false
   317  			if h.forwarding.MaxConcurrency > 0 {
   318  				spawned = h.forwardingBoundWorkers.GoIfAvailable(forward)
   319  			} else {
   320  				go forward()
   321  				spawned = true
   322  			}
   323  			if !spawned {
   324  				h.metrics.forwardDropped.Inc(1)
   325  			}
   326  		}
   327  	}
   328  
   329  	batchErr := h.write(r.Context(), req, opts)
   330  
   331  	// Record ingestion delay latency
   332  	now := h.nowFn()
   333  	for _, series := range req.Timeseries {
   334  		for _, sample := range series.Samples {
   335  			age := now.Sub(storage.PromTimestampToTime(sample.Timestamp))
   336  			h.metrics.ingestLatency.RecordDuration(age)
   337  		}
   338  	}
   339  
   340  	if batchErr != nil {
   341  		var (
   342  			errs                 = batchErr.Errors()
   343  			lastRegularErr       string
   344  			lastBadRequestErr    string
   345  			numRegular           int
   346  			numBadRequest        int
   347  			numResourceExhausted int
   348  		)
   349  		for _, err := range errs {
   350  			switch {
   351  			case client.IsResourceExhaustedError(err):
   352  				numResourceExhausted++
   353  				lastBadRequestErr = err.Error()
   354  			case client.IsBadRequestError(err):
   355  				numBadRequest++
   356  				lastBadRequestErr = err.Error()
   357  			case xerrors.IsInvalidParams(err):
   358  				numBadRequest++
   359  				lastBadRequestErr = err.Error()
   360  			default:
   361  				numRegular++
   362  				lastRegularErr = err.Error()
   363  			}
   364  		}
   365  
   366  		var status int
   367  		switch {
   368  		case numBadRequest == len(errs):
   369  			status = http.StatusBadRequest
   370  		case numResourceExhausted > 0:
   371  			status = http.StatusTooManyRequests
   372  		default:
   373  			status = http.StatusInternalServerError
   374  		}
   375  
   376  		logger := logging.WithContext(r.Context(), h.instrumentOpts)
   377  		logger.Error("write error",
   378  			zap.String("remoteAddr", r.RemoteAddr),
   379  			zap.Int("httpResponseStatusCode", status),
   380  			zap.Int("numResourceExhaustedErrors", numResourceExhausted),
   381  			zap.Int("numRegularErrors", numRegular),
   382  			zap.Int("numBadRequestErrors", numBadRequest),
   383  			zap.String("lastRegularError", lastRegularErr),
   384  			zap.String("lastBadRequestErr", lastBadRequestErr))
   385  
   386  		var resultErrMessage string
   387  		if lastRegularErr != "" {
   388  			resultErrMessage = fmt.Sprintf("retryable_errors: count=%d, last=%s",
   389  				numRegular, lastRegularErr)
   390  		}
   391  		if lastBadRequestErr != "" {
   392  			var sep string
   393  			if lastRegularErr != "" {
   394  				sep = ", "
   395  			}
   396  			resultErrMessage = fmt.Sprintf("%s%sbad_request_errors: count=%d, last=%s",
   397  				resultErrMessage, sep, numBadRequest, lastBadRequestErr)
   398  		}
   399  
   400  		resultError := xhttp.NewError(errors.New(resultErrMessage), status)
   401  		h.metrics.incError(resultError)
   402  		xhttp.WriteError(w, resultError)
   403  		return
   404  	}
   405  
   406  	// NB(schallert): this is frustrating but if we don't explicitly write an HTTP
   407  	// status code (or via Write()), OpenTracing middleware reports code=0 and
   408  	// shows up as error.
   409  	w.WriteHeader(200)
   410  	h.metrics.writeSuccess.Inc(1)
   411  }
   412  
   413  type parseRequestResult struct {
   414  	Request        *prompb.WriteRequest
   415  	Options        ingest.WriteOptions
   416  	CompressResult prometheus.ParsePromCompressedRequestResult
   417  }
   418  
   419  func (h *PromWriteHandler) checkedParseRequest(
   420  	r *http.Request,
   421  ) (parseRequestResult, error) {
   422  	result, err := h.parseRequest(r)
   423  	if err != nil {
   424  		// Always invalid request if parsing fails params.
   425  		return parseRequestResult{}, xerrors.NewInvalidParamsError(err)
   426  	}
   427  	return result, nil
   428  }
   429  
   430  // parseRequest extracts the Prometheus write request from the request body and
   431  // headers. WARNING: it is not guaranteed that the tags returned in the request
   432  // body are in sorted order. It is expected that the caller ensures the tags are
   433  // sorted before passing them to storage, which currently happens in write() ->
   434  // newTSPromIter() -> storage.PromLabelsToM3Tags() -> tags.AddTags(). This is
   435  // the only path written metrics are processed, but future write paths must
   436  // uphold the same guarantees.
   437  func (h *PromWriteHandler) parseRequest(
   438  	r *http.Request,
   439  ) (parseRequestResult, error) {
   440  	var opts ingest.WriteOptions
   441  	if v := strings.TrimSpace(r.Header.Get(headers.MetricsTypeHeader)); v != "" {
   442  		// Allow the metrics type and storage policies to override
   443  		// the default rules and policies if specified.
   444  		metricsType, err := storagemetadata.ParseMetricsType(v)
   445  		if err != nil {
   446  			return parseRequestResult{}, err
   447  		}
   448  
   449  		// Ensure ingest options specify we are overriding the
   450  		// downsampling rules with zero rules to be applied (so
   451  		// only direct writes will be made).
   452  		opts.DownsampleOverride = true
   453  		opts.DownsampleMappingRules = nil
   454  
   455  		strPolicy := strings.TrimSpace(r.Header.Get(headers.MetricsStoragePolicyHeader))
   456  		switch metricsType {
   457  		case storagemetadata.UnaggregatedMetricsType:
   458  			if strPolicy != emptyStoragePolicyVar {
   459  				return parseRequestResult{}, errUnaggregatedStoragePolicySet
   460  			}
   461  		default:
   462  			parsed, err := policy.ParseStoragePolicy(strPolicy)
   463  			if err != nil {
   464  				err = fmt.Errorf("could not parse storage policy: %v", err)
   465  				return parseRequestResult{}, err
   466  			}
   467  
   468  			// Make sure this specific storage policy is used for the writes.
   469  			opts.WriteOverride = true
   470  			opts.WriteStoragePolicies = policy.StoragePolicies{
   471  				parsed,
   472  			}
   473  		}
   474  	}
   475  	if v := strings.TrimSpace(r.Header.Get(headers.WriteTypeHeader)); v != "" {
   476  		switch v {
   477  		case headers.DefaultWriteType:
   478  		case headers.AggregateWriteType:
   479  			opts.WriteOverride = true
   480  			opts.WriteStoragePolicies = policy.StoragePolicies{}
   481  		default:
   482  			err := fmt.Errorf("unrecognized write type: %s", v)
   483  			return parseRequestResult{}, err
   484  		}
   485  	}
   486  
   487  	result, err := prometheus.ParsePromCompressedRequest(r)
   488  	if err != nil {
   489  		return parseRequestResult{}, err
   490  	}
   491  
   492  	var req prompb.WriteRequest
   493  	if err := proto.Unmarshal(result.UncompressedBody, &req); err != nil {
   494  		return parseRequestResult{}, err
   495  	}
   496  
   497  	if mapStr := r.Header.Get(headers.MapTagsByJSONHeader); mapStr != "" {
   498  		var opts handleroptions.MapTagsOptions
   499  		if err := json.Unmarshal([]byte(mapStr), &opts); err != nil {
   500  			return parseRequestResult{}, err
   501  		}
   502  
   503  		if err := mapTags(&req, opts); err != nil {
   504  			return parseRequestResult{}, err
   505  		}
   506  	}
   507  
   508  	if promType := r.Header.Get(headers.PromTypeHeader); promType != "" {
   509  		tp, ok := headerToMetricType[strings.ToLower(promType)]
   510  		if !ok {
   511  			return parseRequestResult{}, fmt.Errorf("unknown prom metric type %s", promType)
   512  		}
   513  		for i := range req.Timeseries {
   514  			req.Timeseries[i].Type = tp
   515  		}
   516  	}
   517  
   518  	// Check if any of the labels exceed literal length limits and occasionally print them
   519  	// in a log message for debugging purposes.
   520  	maxTagLiteralLength := int(h.tagOptions.MaxTagLiteralLength())
   521  	for _, ts := range req.Timeseries {
   522  		for _, l := range ts.Labels {
   523  			if len(l.Name) > maxTagLiteralLength || len(l.Value) > maxTagLiteralLength {
   524  				h.maybeLogLabelsWithTooLongLiterals(h.instrumentOpts.Logger(), l)
   525  				err := fmt.Errorf("label literal is too long: nameLength=%d, valueLength=%d, maxLength=%d",
   526  					len(l.Name), len(l.Value), maxTagLiteralLength)
   527  				return parseRequestResult{}, err
   528  			}
   529  		}
   530  	}
   531  
   532  	return parseRequestResult{
   533  		Request:        &req,
   534  		Options:        opts,
   535  		CompressResult: result,
   536  	}, nil
   537  }
   538  
   539  func (h *PromWriteHandler) write(
   540  	ctx context.Context,
   541  	r *prompb.WriteRequest,
   542  	opts ingest.WriteOptions,
   543  ) ingest.BatchError {
   544  	iter, err := newPromTSIter(r.Timeseries, h.tagOptions, h.storeMetricsType)
   545  	if err != nil {
   546  		var errs xerrors.MultiError
   547  		return errs.Add(err)
   548  	}
   549  	return h.downsamplerAndWriter.WriteBatch(ctx, iter, opts)
   550  }
   551  
   552  func (h *PromWriteHandler) forward(
   553  	ctx context.Context,
   554  	res parseRequestResult,
   555  	header http.Header,
   556  	target handleroptions.PromWriteHandlerForwardTargetOptions,
   557  ) error {
   558  	body := bytes.NewReader(res.CompressResult.CompressedBody)
   559  	if shadowOpts := target.Shadow; shadowOpts != nil {
   560  		// Need to send a subset of the original series to the shadow target.
   561  		buffer, err := h.buildForwardShadowRequestBody(res, shadowOpts)
   562  		if err != nil {
   563  			return err
   564  		}
   565  		// Read the body from the shadow request body just built.
   566  		body.Reset(buffer)
   567  	}
   568  
   569  	method := target.Method
   570  	if method == "" {
   571  		method = http.MethodPost
   572  	}
   573  	url := target.URL
   574  	req, err := http.NewRequest(method, url, body)
   575  	if err != nil {
   576  		return err
   577  	}
   578  
   579  	// There are multiple headers that impact coordinator behavior on the write
   580  	// (map tags, storage policy, etc.) that we must forward to the target
   581  	// coordinator to guarantee same behavior as the coordinator that originally
   582  	// received the request.
   583  	if header != nil {
   584  		for h := range header {
   585  			if strings.HasPrefix(h, headers.M3HeaderPrefix) {
   586  				req.Header.Add(h, header.Get(h))
   587  			}
   588  		}
   589  	}
   590  
   591  	if targetHeaders := target.Headers; targetHeaders != nil {
   592  		// If headers set, attach to request.
   593  		for name, value := range targetHeaders {
   594  			req.Header.Add(name, value)
   595  		}
   596  	}
   597  
   598  	resp, err := h.forwardHTTPClient.Do(req.WithContext(ctx))
   599  	if err != nil {
   600  		return err
   601  	}
   602  
   603  	defer resp.Body.Close()
   604  
   605  	if resp.StatusCode/100 != 2 {
   606  		response, err := ioutil.ReadAll(resp.Body)
   607  		if err != nil {
   608  			response = []byte(fmt.Sprintf("error reading body: %v", err))
   609  		}
   610  		return fmt.Errorf("expected status code 2XX: actual=%v, method=%v, url=%v, resp=%s",
   611  			resp.StatusCode, method, url, response)
   612  	}
   613  
   614  	return nil
   615  }
   616  
   617  func (h *PromWriteHandler) buildForwardShadowRequestBody(
   618  	res parseRequestResult,
   619  	shadowOpts *handleroptions.PromWriteHandlerForwardTargetShadowOptions,
   620  ) ([]byte, error) {
   621  	if shadowOpts.Percent < 0 || shadowOpts.Percent > 1 {
   622  		return nil, fmt.Errorf("forwarding shadow percent out of range [0,1]: %f",
   623  			shadowOpts.Percent)
   624  	}
   625  
   626  	// Need to apply shadow percent.
   627  	var hash func([]byte) uint64
   628  	switch shadowOpts.Hash {
   629  	case "":
   630  		fallthrough
   631  	case "xxhash":
   632  		hash = xxhash.Sum64
   633  	case "murmur3":
   634  		hash = murmur3.Sum64
   635  	default:
   636  		return nil, fmt.Errorf("unknown hash function: %s", shadowOpts.Hash)
   637  	}
   638  
   639  	var (
   640  		shadowReq = &prompb.WriteRequest{}
   641  		labels    []prompb.Label
   642  		buffer    []byte
   643  	)
   644  	for _, ts := range res.Request.Timeseries {
   645  		// Build an ID of the series to hash.
   646  		// First take copy of labels so the call to sort doesn't modify the
   647  		// original slice.
   648  		labels = append(labels[:0], ts.Labels...)
   649  		buffer = buildPseudoIDWithLabelsLikelySorted(labels, buffer[:0])
   650  
   651  		// Use a range of 10k to allow for setting 0.01% having an effect
   652  		// when shadow percent is set (i.e. with percent=0.0001)
   653  		if hash(buffer)%10000 >= uint64(shadowOpts.Percent*10000) {
   654  			// Skip forwarding this series, not in shadow volume of shards.
   655  			// Swap it with the tail and continue.
   656  			h.metrics.forwardShadowDrop.Inc(1)
   657  			continue
   658  		}
   659  
   660  		// Keep this series, it falls below the volume target of shards.
   661  		h.metrics.forwardShadowKeep.Inc(1)
   662  
   663  		// Skip forwarding this series, not in shadow volume of shards.
   664  		// Swap it with the tail and continue.
   665  		shadowReq.Timeseries = append(shadowReq.Timeseries, ts)
   666  	}
   667  
   668  	encoded, err := proto.Marshal(shadowReq)
   669  	if err != nil {
   670  		return nil, fmt.Errorf("failed to marshal forwarding shadow request: %w", err)
   671  	}
   672  
   673  	return snappy.Encode(buffer[:0], encoded), nil
   674  }
   675  
   676  // buildPseudoIDWithLabelsLikelySorted will build a pseudo ID that can be
   677  // hashed/etc (but not used as primary key since not escaped), it expects the
   678  // input labels to be likely sorted (so can avoid invoking sort in the regular
   679  // case where series have labels already sorted when sent to remote write
   680  // endpoint, which is commonly the case).
   681  func buildPseudoIDWithLabelsLikelySorted(
   682  	labels []prompb.Label,
   683  	buffer []byte,
   684  ) []byte {
   685  	for i, l := range labels {
   686  		if i > 0 && bytes.Compare(l.Name, labels[i-1].Name) < 0 {
   687  			// Sort.
   688  			sort.Sort(sortableLabels(labels))
   689  			// Rebuild.
   690  			return buildPseudoIDWithLabelsLikelySorted(labels, buffer[:0])
   691  		}
   692  		buffer = append(buffer, l.Name...)
   693  		buffer = append(buffer, '=')
   694  		buffer = append(buffer, l.Value...)
   695  		if i < len(labels)-1 {
   696  			buffer = append(buffer, ',')
   697  		}
   698  	}
   699  	return buffer
   700  }
   701  
   702  func (h *PromWriteHandler) maybeLogLabelsWithTooLongLiterals(logger *zap.Logger, label prompb.Label) {
   703  	if atomic.AddUint32(&h.numLiteralIsTooLong, 1) > maxLiteralIsTooLongLogCount {
   704  		return
   705  	}
   706  
   707  	safePrefix := func(b []byte, l int) []byte {
   708  		if len(b) <= l {
   709  			return b
   710  		}
   711  		return b[:l]
   712  	}
   713  
   714  	logger.Warn("label exceeds literal length limits",
   715  		zap.String("namePrefix", string(safePrefix(label.Name, literalPrefixLength))),
   716  		zap.Int("nameLength", len(label.Name)),
   717  		zap.String("valuePrefix", string(safePrefix(label.Value, literalPrefixLength))),
   718  		zap.Int("valueLength", len(label.Value)),
   719  	)
   720  }
   721  
   722  func newPromTSIter(
   723  	timeseries []prompb.TimeSeries,
   724  	tagOpts models.TagOptions,
   725  	storeMetricsType bool,
   726  ) (*promTSIter, error) {
   727  	// Construct the tags and datapoints upfront so that if the iterator
   728  	// is reset, we don't have to generate them twice.
   729  	var (
   730  		tags             = make([]models.Tags, 0, len(timeseries))
   731  		datapoints       = make([]ts.Datapoints, 0, len(timeseries))
   732  		seriesAttributes = make([]ts.SeriesAttributes, 0, len(timeseries))
   733  	)
   734  
   735  	graphiteTagOpts := tagOpts.SetIDSchemeType(models.TypeGraphite)
   736  	for _, promTS := range timeseries {
   737  		attributes, err := storage.PromTimeSeriesToSeriesAttributes(promTS)
   738  		if err != nil {
   739  			return nil, err
   740  		}
   741  
   742  		// Set the tag options based on the incoming source.
   743  		opts := tagOpts
   744  		if attributes.Source == ts.SourceTypeGraphite {
   745  			opts = graphiteTagOpts
   746  		}
   747  
   748  		seriesAttributes = append(seriesAttributes, attributes)
   749  		tags = append(tags, storage.PromLabelsToM3Tags(promTS.Labels, opts))
   750  		datapoints = append(datapoints, storage.PromSamplesToM3Datapoints(promTS.Samples))
   751  	}
   752  
   753  	return &promTSIter{
   754  		attributes:       seriesAttributes,
   755  		idx:              -1,
   756  		tags:             tags,
   757  		datapoints:       datapoints,
   758  		storeMetricsType: storeMetricsType,
   759  	}, nil
   760  }
   761  
   762  type promTSIter struct {
   763  	idx        int
   764  	err        error
   765  	attributes []ts.SeriesAttributes
   766  	tags       []models.Tags
   767  	datapoints []ts.Datapoints
   768  	metadatas  []ts.Metadata
   769  	annotation []byte
   770  
   771  	storeMetricsType bool
   772  }
   773  
   774  func (i *promTSIter) Next() bool {
   775  	if i.err != nil {
   776  		return false
   777  	}
   778  
   779  	i.idx++
   780  	if i.idx >= len(i.tags) {
   781  		return false
   782  	}
   783  
   784  	if !i.storeMetricsType {
   785  		return true
   786  	}
   787  
   788  	annotationPayload, err := storage.SeriesAttributesToAnnotationPayload(i.attributes[i.idx])
   789  	if err != nil {
   790  		i.err = err
   791  		return false
   792  	}
   793  
   794  	i.annotation, err = annotationPayload.Marshal()
   795  	if err != nil {
   796  		i.err = err
   797  		return false
   798  	}
   799  
   800  	if len(i.annotation) == 0 {
   801  		i.annotation = nil
   802  	}
   803  
   804  	return true
   805  }
   806  
   807  func (i *promTSIter) Current() ingest.IterValue {
   808  	if len(i.tags) == 0 || i.idx < 0 || i.idx >= len(i.tags) {
   809  		return defaultValue
   810  	}
   811  
   812  	value := ingest.IterValue{
   813  		Tags:       i.tags[i.idx],
   814  		Datapoints: i.datapoints[i.idx],
   815  		Attributes: i.attributes[i.idx],
   816  		Unit:       xtime.Millisecond,
   817  		Annotation: i.annotation,
   818  	}
   819  	if i.idx < len(i.metadatas) {
   820  		value.Metadata = i.metadatas[i.idx]
   821  	}
   822  	return value
   823  }
   824  
   825  func (i *promTSIter) Reset() error {
   826  	i.idx = -1
   827  	i.err = nil
   828  	i.annotation = nil
   829  
   830  	return nil
   831  }
   832  
   833  func (i *promTSIter) Error() error {
   834  	return i.err
   835  }
   836  
   837  func (i *promTSIter) SetCurrentMetadata(metadata ts.Metadata) {
   838  	if len(i.metadatas) == 0 {
   839  		i.metadatas = make([]ts.Metadata, len(i.tags))
   840  	}
   841  	if i.idx < 0 || i.idx >= len(i.metadatas) {
   842  		return
   843  	}
   844  	i.metadatas[i.idx] = metadata
   845  }
   846  
   847  type sortableLabels []prompb.Label
   848  
   849  func (t sortableLabels) Len() int      { return len(t) }
   850  func (t sortableLabels) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
   851  func (t sortableLabels) Less(i, j int) bool {
   852  	return bytes.Compare(t[i].Name, t[j].Name) == -1
   853  }