github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/query/api/v1/handler/prometheus/remote/write.go (about) 1 // Copyright (c) 2018 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package remote 22 23 import ( 24 "bytes" 25 "context" 26 "encoding/json" 27 "errors" 28 "fmt" 29 "io/ioutil" 30 "net/http" 31 "sort" 32 "strings" 33 "sync/atomic" 34 "time" 35 36 "github.com/m3db/m3/src/cmd/services/m3coordinator/ingest" 37 "github.com/m3db/m3/src/dbnode/client" 38 "github.com/m3db/m3/src/metrics/policy" 39 "github.com/m3db/m3/src/query/api/v1/handler/prometheus" 40 "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" 41 "github.com/m3db/m3/src/query/api/v1/options" 42 "github.com/m3db/m3/src/query/api/v1/route" 43 "github.com/m3db/m3/src/query/generated/proto/prompb" 44 "github.com/m3db/m3/src/query/models" 45 "github.com/m3db/m3/src/query/storage" 46 "github.com/m3db/m3/src/query/storage/m3/storagemetadata" 47 "github.com/m3db/m3/src/query/ts" 48 "github.com/m3db/m3/src/query/util/logging" 49 "github.com/m3db/m3/src/x/clock" 50 xerrors "github.com/m3db/m3/src/x/errors" 51 "github.com/m3db/m3/src/x/headers" 52 "github.com/m3db/m3/src/x/instrument" 53 xhttp "github.com/m3db/m3/src/x/net/http" 54 "github.com/m3db/m3/src/x/retry" 55 xsync "github.com/m3db/m3/src/x/sync" 56 xtime "github.com/m3db/m3/src/x/time" 57 58 "github.com/cespare/xxhash/v2" 59 "github.com/golang/protobuf/proto" 60 "github.com/golang/snappy" 61 murmur3 "github.com/m3db/stackmurmur3/v2" 62 "github.com/uber-go/tally" 63 "go.uber.org/zap" 64 ) 65 66 const ( 67 // PromWriteURL is the url for the prom write handler 68 PromWriteURL = route.Prefix + "/prom/remote/write" 69 70 // PromWriteHTTPMethod is the HTTP method used with this resource. 71 PromWriteHTTPMethod = http.MethodPost 72 73 // emptyStoragePolicyVar for code readability. 74 emptyStoragePolicyVar = "" 75 76 // defaultForwardingTimeout is the default forwarding timeout. 77 defaultForwardingTimeout = 15 * time.Second 78 79 // maxLiteralIsTooLongLogCount is the number of times the time series labels should be logged 80 // upon "literal is too long" error. 81 maxLiteralIsTooLongLogCount = 10 82 // literalPrefixLength is the length of the label literal prefix that is logged upon 83 // "literal is too long" error. 84 literalPrefixLength = 100 85 ) 86 87 var ( 88 errNoDownsamplerAndWriter = errors.New("no downsampler and writer set") 89 errNoTagOptions = errors.New("no tag options set") 90 errNoNowFn = errors.New("no now fn set") 91 errUnaggregatedStoragePolicySet = errors.New("storage policy should not be set for unaggregated metrics") 92 93 defaultForwardingRetryForever = false 94 defaultForwardingRetryJitter = true 95 defaultForwardRetryConfig = retry.Configuration{ 96 InitialBackoff: time.Second * 2, 97 BackoffFactor: 2, 98 MaxRetries: 1, 99 Forever: &defaultForwardingRetryForever, 100 Jitter: &defaultForwardingRetryJitter, 101 } 102 103 defaultValue = ingest.IterValue{ 104 Tags: models.EmptyTags(), 105 Attributes: ts.DefaultSeriesAttributes(), 106 Metadata: ts.Metadata{}, 107 } 108 109 headerToMetricType = map[string]prompb.MetricType{ 110 "counter": prompb.MetricType_COUNTER, 111 "gauge": prompb.MetricType_GAUGE, 112 "gauge_histogram": prompb.MetricType_GAUGE_HISTOGRAM, 113 "histogram": prompb.MetricType_HISTOGRAM, 114 "info": prompb.MetricType_INFO, 115 "stateset": prompb.MetricType_STATESET, 116 "summary": prompb.MetricType_SUMMARY, 117 } 118 ) 119 120 // PromWriteHandler represents a handler for prometheus write endpoint. 121 type PromWriteHandler struct { 122 downsamplerAndWriter ingest.DownsamplerAndWriter 123 tagOptions models.TagOptions 124 storeMetricsType bool 125 forwarding handleroptions.PromWriteHandlerForwardingOptions 126 forwardTimeout time.Duration 127 forwardHTTPClient *http.Client 128 forwardingBoundWorkers xsync.WorkerPool 129 forwardContext context.Context 130 forwardRetrier retry.Retrier 131 nowFn clock.NowFn 132 instrumentOpts instrument.Options 133 metrics promWriteMetrics 134 135 // Counting the number of times of "literal is too long" error for log sampling purposes. 136 numLiteralIsTooLong uint32 137 } 138 139 // NewPromWriteHandler returns a new instance of handler. 140 func NewPromWriteHandler(options options.HandlerOptions) (http.Handler, error) { 141 var ( 142 downsamplerAndWriter = options.DownsamplerAndWriter() 143 tagOptions = options.TagOptions() 144 nowFn = options.NowFn() 145 forwarding = options.Config().WriteForwarding.PromRemoteWrite 146 instrumentOpts = options.InstrumentOpts() 147 ) 148 149 if downsamplerAndWriter == nil { 150 return nil, errNoDownsamplerAndWriter 151 } 152 153 if tagOptions == nil { 154 return nil, errNoTagOptions 155 } 156 157 if nowFn == nil { 158 return nil, errNoNowFn 159 } 160 161 scope := options.InstrumentOpts(). 162 MetricsScope(). 163 Tagged(map[string]string{"handler": "remote-write"}) 164 metrics, err := newPromWriteMetrics(scope) 165 if err != nil { 166 return nil, err 167 } 168 169 // Only use a forwarding worker pool if concurrency is bound, otherwise 170 // if unlimited we just spin up a goroutine for each incoming write. 171 var forwardingBoundWorkers xsync.WorkerPool 172 if v := forwarding.MaxConcurrency; v > 0 { 173 forwardingBoundWorkers = xsync.NewWorkerPool(v) 174 forwardingBoundWorkers.Init() 175 } 176 177 forwardTimeout := defaultForwardingTimeout 178 if v := forwarding.Timeout; v > 0 { 179 forwardTimeout = v 180 } 181 182 forwardHTTPOpts := xhttp.DefaultHTTPClientOptions() 183 forwardHTTPOpts.DisableCompression = true // Already snappy compressed. 184 forwardHTTPOpts.RequestTimeout = forwardTimeout 185 186 forwardRetryConfig := defaultForwardRetryConfig 187 if forwarding.Retry != nil { 188 forwardRetryConfig = *forwarding.Retry 189 } 190 forwardRetryOpts := forwardRetryConfig.NewOptions( 191 scope.SubScope("forwarding-retry"), 192 ) 193 194 return &PromWriteHandler{ 195 downsamplerAndWriter: downsamplerAndWriter, 196 tagOptions: tagOptions, 197 storeMetricsType: options.StoreMetricsType(), 198 forwarding: forwarding, 199 forwardTimeout: forwardTimeout, 200 forwardHTTPClient: xhttp.NewHTTPClient(forwardHTTPOpts), 201 forwardingBoundWorkers: forwardingBoundWorkers, 202 forwardContext: context.Background(), 203 forwardRetrier: retry.NewRetrier(forwardRetryOpts), 204 nowFn: nowFn, 205 metrics: metrics, 206 instrumentOpts: instrumentOpts, 207 }, nil 208 } 209 210 type promWriteMetrics struct { 211 writeSuccess tally.Counter 212 writeErrorsServer tally.Counter 213 writeErrorsClient tally.Counter 214 writeBatchLatency tally.Histogram 215 writeBatchLatencyBuckets tally.DurationBuckets 216 ingestLatency tally.Histogram 217 ingestLatencyBuckets tally.DurationBuckets 218 forwardSuccess tally.Counter 219 forwardErrors tally.Counter 220 forwardDropped tally.Counter 221 forwardLatency tally.Histogram 222 forwardShadowKeep tally.Counter 223 forwardShadowDrop tally.Counter 224 } 225 226 func (m *promWriteMetrics) incError(err error) { 227 if xhttp.IsClientError(err) { 228 m.writeErrorsClient.Inc(1) 229 } else { 230 m.writeErrorsServer.Inc(1) 231 } 232 } 233 234 func newPromWriteMetrics(scope tally.Scope) (promWriteMetrics, error) { 235 buckets, err := ingest.NewLatencyBuckets() 236 if err != nil { 237 return promWriteMetrics{}, err 238 } 239 return promWriteMetrics{ 240 writeSuccess: scope.SubScope("write").Counter("success"), 241 writeErrorsServer: scope.SubScope("write").Tagged(map[string]string{"code": "5XX"}).Counter("errors"), 242 writeErrorsClient: scope.SubScope("write").Tagged(map[string]string{"code": "4XX"}).Counter("errors"), 243 writeBatchLatency: scope.SubScope("write").Histogram("batch-latency", buckets.WriteLatencyBuckets), 244 writeBatchLatencyBuckets: buckets.WriteLatencyBuckets, 245 ingestLatency: scope.SubScope("ingest").Histogram("latency", buckets.IngestLatencyBuckets), 246 ingestLatencyBuckets: buckets.IngestLatencyBuckets, 247 forwardSuccess: scope.SubScope("forward").Counter("success"), 248 forwardErrors: scope.SubScope("forward").Counter("errors"), 249 forwardDropped: scope.SubScope("forward").Counter("dropped"), 250 forwardLatency: scope.SubScope("forward").Histogram("latency", buckets.WriteLatencyBuckets), 251 forwardShadowKeep: scope.SubScope("forward").SubScope("shadow").Counter("keep"), 252 forwardShadowDrop: scope.SubScope("forward").SubScope("shadow").Counter("drop"), 253 }, nil 254 } 255 256 func (h *PromWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 257 batchRequestStopwatch := h.metrics.writeBatchLatency.Start() 258 defer batchRequestStopwatch.Stop() 259 260 checkedReq, err := h.checkedParseRequest(r) 261 if err != nil { 262 h.metrics.incError(err) 263 xhttp.WriteError(w, err) 264 return 265 } 266 267 var ( 268 req = checkedReq.Request 269 opts = checkedReq.Options 270 ) 271 // Begin async forwarding. 272 // NB(r): Be careful about not returning buffers to pool 273 // if the request bodies ever get pooled until after 274 // forwarding completes. 275 if targets := h.forwarding.Targets; len(targets) > 0 { 276 for _, target := range targets { 277 target := target // Capture for lambda. 278 forward := func() { 279 now := h.nowFn() 280 281 var ( 282 attempt = func() error { 283 // Consider propagating baggage without tying 284 // context to request context in future. 285 ctx, cancel := context.WithTimeout(h.forwardContext, h.forwardTimeout) 286 defer cancel() 287 return h.forward(ctx, checkedReq, r.Header, target) 288 } 289 err error 290 ) 291 if target.NoRetry { 292 err = attempt() 293 } else { 294 err = h.forwardRetrier.Attempt(attempt) 295 } 296 297 // Record forward ingestion delay. 298 // NB: this includes any time for retries. 299 for _, series := range req.Timeseries { 300 for _, sample := range series.Samples { 301 age := now.Sub(storage.PromTimestampToTime(sample.Timestamp)) 302 h.metrics.forwardLatency.RecordDuration(age) 303 } 304 } 305 306 if err != nil { 307 h.metrics.forwardErrors.Inc(1) 308 logger := logging.WithContext(h.forwardContext, h.instrumentOpts) 309 logger.Error("forward error", zap.Error(err)) 310 return 311 } 312 313 h.metrics.forwardSuccess.Inc(1) 314 } 315 316 spawned := false 317 if h.forwarding.MaxConcurrency > 0 { 318 spawned = h.forwardingBoundWorkers.GoIfAvailable(forward) 319 } else { 320 go forward() 321 spawned = true 322 } 323 if !spawned { 324 h.metrics.forwardDropped.Inc(1) 325 } 326 } 327 } 328 329 batchErr := h.write(r.Context(), req, opts) 330 331 // Record ingestion delay latency 332 now := h.nowFn() 333 for _, series := range req.Timeseries { 334 for _, sample := range series.Samples { 335 age := now.Sub(storage.PromTimestampToTime(sample.Timestamp)) 336 h.metrics.ingestLatency.RecordDuration(age) 337 } 338 } 339 340 if batchErr != nil { 341 var ( 342 errs = batchErr.Errors() 343 lastRegularErr string 344 lastBadRequestErr string 345 numRegular int 346 numBadRequest int 347 numResourceExhausted int 348 ) 349 for _, err := range errs { 350 switch { 351 case client.IsResourceExhaustedError(err): 352 numResourceExhausted++ 353 lastBadRequestErr = err.Error() 354 case client.IsBadRequestError(err): 355 numBadRequest++ 356 lastBadRequestErr = err.Error() 357 case xerrors.IsInvalidParams(err): 358 numBadRequest++ 359 lastBadRequestErr = err.Error() 360 default: 361 numRegular++ 362 lastRegularErr = err.Error() 363 } 364 } 365 366 var status int 367 switch { 368 case numBadRequest == len(errs): 369 status = http.StatusBadRequest 370 case numResourceExhausted > 0: 371 status = http.StatusTooManyRequests 372 default: 373 status = http.StatusInternalServerError 374 } 375 376 logger := logging.WithContext(r.Context(), h.instrumentOpts) 377 logger.Error("write error", 378 zap.String("remoteAddr", r.RemoteAddr), 379 zap.Int("httpResponseStatusCode", status), 380 zap.Int("numResourceExhaustedErrors", numResourceExhausted), 381 zap.Int("numRegularErrors", numRegular), 382 zap.Int("numBadRequestErrors", numBadRequest), 383 zap.String("lastRegularError", lastRegularErr), 384 zap.String("lastBadRequestErr", lastBadRequestErr)) 385 386 var resultErrMessage string 387 if lastRegularErr != "" { 388 resultErrMessage = fmt.Sprintf("retryable_errors: count=%d, last=%s", 389 numRegular, lastRegularErr) 390 } 391 if lastBadRequestErr != "" { 392 var sep string 393 if lastRegularErr != "" { 394 sep = ", " 395 } 396 resultErrMessage = fmt.Sprintf("%s%sbad_request_errors: count=%d, last=%s", 397 resultErrMessage, sep, numBadRequest, lastBadRequestErr) 398 } 399 400 resultError := xhttp.NewError(errors.New(resultErrMessage), status) 401 h.metrics.incError(resultError) 402 xhttp.WriteError(w, resultError) 403 return 404 } 405 406 // NB(schallert): this is frustrating but if we don't explicitly write an HTTP 407 // status code (or via Write()), OpenTracing middleware reports code=0 and 408 // shows up as error. 409 w.WriteHeader(200) 410 h.metrics.writeSuccess.Inc(1) 411 } 412 413 type parseRequestResult struct { 414 Request *prompb.WriteRequest 415 Options ingest.WriteOptions 416 CompressResult prometheus.ParsePromCompressedRequestResult 417 } 418 419 func (h *PromWriteHandler) checkedParseRequest( 420 r *http.Request, 421 ) (parseRequestResult, error) { 422 result, err := h.parseRequest(r) 423 if err != nil { 424 // Always invalid request if parsing fails params. 425 return parseRequestResult{}, xerrors.NewInvalidParamsError(err) 426 } 427 return result, nil 428 } 429 430 // parseRequest extracts the Prometheus write request from the request body and 431 // headers. WARNING: it is not guaranteed that the tags returned in the request 432 // body are in sorted order. It is expected that the caller ensures the tags are 433 // sorted before passing them to storage, which currently happens in write() -> 434 // newTSPromIter() -> storage.PromLabelsToM3Tags() -> tags.AddTags(). This is 435 // the only path written metrics are processed, but future write paths must 436 // uphold the same guarantees. 437 func (h *PromWriteHandler) parseRequest( 438 r *http.Request, 439 ) (parseRequestResult, error) { 440 var opts ingest.WriteOptions 441 if v := strings.TrimSpace(r.Header.Get(headers.MetricsTypeHeader)); v != "" { 442 // Allow the metrics type and storage policies to override 443 // the default rules and policies if specified. 444 metricsType, err := storagemetadata.ParseMetricsType(v) 445 if err != nil { 446 return parseRequestResult{}, err 447 } 448 449 // Ensure ingest options specify we are overriding the 450 // downsampling rules with zero rules to be applied (so 451 // only direct writes will be made). 452 opts.DownsampleOverride = true 453 opts.DownsampleMappingRules = nil 454 455 strPolicy := strings.TrimSpace(r.Header.Get(headers.MetricsStoragePolicyHeader)) 456 switch metricsType { 457 case storagemetadata.UnaggregatedMetricsType: 458 if strPolicy != emptyStoragePolicyVar { 459 return parseRequestResult{}, errUnaggregatedStoragePolicySet 460 } 461 default: 462 parsed, err := policy.ParseStoragePolicy(strPolicy) 463 if err != nil { 464 err = fmt.Errorf("could not parse storage policy: %v", err) 465 return parseRequestResult{}, err 466 } 467 468 // Make sure this specific storage policy is used for the writes. 469 opts.WriteOverride = true 470 opts.WriteStoragePolicies = policy.StoragePolicies{ 471 parsed, 472 } 473 } 474 } 475 if v := strings.TrimSpace(r.Header.Get(headers.WriteTypeHeader)); v != "" { 476 switch v { 477 case headers.DefaultWriteType: 478 case headers.AggregateWriteType: 479 opts.WriteOverride = true 480 opts.WriteStoragePolicies = policy.StoragePolicies{} 481 default: 482 err := fmt.Errorf("unrecognized write type: %s", v) 483 return parseRequestResult{}, err 484 } 485 } 486 487 result, err := prometheus.ParsePromCompressedRequest(r) 488 if err != nil { 489 return parseRequestResult{}, err 490 } 491 492 var req prompb.WriteRequest 493 if err := proto.Unmarshal(result.UncompressedBody, &req); err != nil { 494 return parseRequestResult{}, err 495 } 496 497 if mapStr := r.Header.Get(headers.MapTagsByJSONHeader); mapStr != "" { 498 var opts handleroptions.MapTagsOptions 499 if err := json.Unmarshal([]byte(mapStr), &opts); err != nil { 500 return parseRequestResult{}, err 501 } 502 503 if err := mapTags(&req, opts); err != nil { 504 return parseRequestResult{}, err 505 } 506 } 507 508 if promType := r.Header.Get(headers.PromTypeHeader); promType != "" { 509 tp, ok := headerToMetricType[strings.ToLower(promType)] 510 if !ok { 511 return parseRequestResult{}, fmt.Errorf("unknown prom metric type %s", promType) 512 } 513 for i := range req.Timeseries { 514 req.Timeseries[i].Type = tp 515 } 516 } 517 518 // Check if any of the labels exceed literal length limits and occasionally print them 519 // in a log message for debugging purposes. 520 maxTagLiteralLength := int(h.tagOptions.MaxTagLiteralLength()) 521 for _, ts := range req.Timeseries { 522 for _, l := range ts.Labels { 523 if len(l.Name) > maxTagLiteralLength || len(l.Value) > maxTagLiteralLength { 524 h.maybeLogLabelsWithTooLongLiterals(h.instrumentOpts.Logger(), l) 525 err := fmt.Errorf("label literal is too long: nameLength=%d, valueLength=%d, maxLength=%d", 526 len(l.Name), len(l.Value), maxTagLiteralLength) 527 return parseRequestResult{}, err 528 } 529 } 530 } 531 532 return parseRequestResult{ 533 Request: &req, 534 Options: opts, 535 CompressResult: result, 536 }, nil 537 } 538 539 func (h *PromWriteHandler) write( 540 ctx context.Context, 541 r *prompb.WriteRequest, 542 opts ingest.WriteOptions, 543 ) ingest.BatchError { 544 iter, err := newPromTSIter(r.Timeseries, h.tagOptions, h.storeMetricsType) 545 if err != nil { 546 var errs xerrors.MultiError 547 return errs.Add(err) 548 } 549 return h.downsamplerAndWriter.WriteBatch(ctx, iter, opts) 550 } 551 552 func (h *PromWriteHandler) forward( 553 ctx context.Context, 554 res parseRequestResult, 555 header http.Header, 556 target handleroptions.PromWriteHandlerForwardTargetOptions, 557 ) error { 558 body := bytes.NewReader(res.CompressResult.CompressedBody) 559 if shadowOpts := target.Shadow; shadowOpts != nil { 560 // Need to send a subset of the original series to the shadow target. 561 buffer, err := h.buildForwardShadowRequestBody(res, shadowOpts) 562 if err != nil { 563 return err 564 } 565 // Read the body from the shadow request body just built. 566 body.Reset(buffer) 567 } 568 569 method := target.Method 570 if method == "" { 571 method = http.MethodPost 572 } 573 url := target.URL 574 req, err := http.NewRequest(method, url, body) 575 if err != nil { 576 return err 577 } 578 579 // There are multiple headers that impact coordinator behavior on the write 580 // (map tags, storage policy, etc.) that we must forward to the target 581 // coordinator to guarantee same behavior as the coordinator that originally 582 // received the request. 583 if header != nil { 584 for h := range header { 585 if strings.HasPrefix(h, headers.M3HeaderPrefix) { 586 req.Header.Add(h, header.Get(h)) 587 } 588 } 589 } 590 591 if targetHeaders := target.Headers; targetHeaders != nil { 592 // If headers set, attach to request. 593 for name, value := range targetHeaders { 594 req.Header.Add(name, value) 595 } 596 } 597 598 resp, err := h.forwardHTTPClient.Do(req.WithContext(ctx)) 599 if err != nil { 600 return err 601 } 602 603 defer resp.Body.Close() 604 605 if resp.StatusCode/100 != 2 { 606 response, err := ioutil.ReadAll(resp.Body) 607 if err != nil { 608 response = []byte(fmt.Sprintf("error reading body: %v", err)) 609 } 610 return fmt.Errorf("expected status code 2XX: actual=%v, method=%v, url=%v, resp=%s", 611 resp.StatusCode, method, url, response) 612 } 613 614 return nil 615 } 616 617 func (h *PromWriteHandler) buildForwardShadowRequestBody( 618 res parseRequestResult, 619 shadowOpts *handleroptions.PromWriteHandlerForwardTargetShadowOptions, 620 ) ([]byte, error) { 621 if shadowOpts.Percent < 0 || shadowOpts.Percent > 1 { 622 return nil, fmt.Errorf("forwarding shadow percent out of range [0,1]: %f", 623 shadowOpts.Percent) 624 } 625 626 // Need to apply shadow percent. 627 var hash func([]byte) uint64 628 switch shadowOpts.Hash { 629 case "": 630 fallthrough 631 case "xxhash": 632 hash = xxhash.Sum64 633 case "murmur3": 634 hash = murmur3.Sum64 635 default: 636 return nil, fmt.Errorf("unknown hash function: %s", shadowOpts.Hash) 637 } 638 639 var ( 640 shadowReq = &prompb.WriteRequest{} 641 labels []prompb.Label 642 buffer []byte 643 ) 644 for _, ts := range res.Request.Timeseries { 645 // Build an ID of the series to hash. 646 // First take copy of labels so the call to sort doesn't modify the 647 // original slice. 648 labels = append(labels[:0], ts.Labels...) 649 buffer = buildPseudoIDWithLabelsLikelySorted(labels, buffer[:0]) 650 651 // Use a range of 10k to allow for setting 0.01% having an effect 652 // when shadow percent is set (i.e. with percent=0.0001) 653 if hash(buffer)%10000 >= uint64(shadowOpts.Percent*10000) { 654 // Skip forwarding this series, not in shadow volume of shards. 655 // Swap it with the tail and continue. 656 h.metrics.forwardShadowDrop.Inc(1) 657 continue 658 } 659 660 // Keep this series, it falls below the volume target of shards. 661 h.metrics.forwardShadowKeep.Inc(1) 662 663 // Skip forwarding this series, not in shadow volume of shards. 664 // Swap it with the tail and continue. 665 shadowReq.Timeseries = append(shadowReq.Timeseries, ts) 666 } 667 668 encoded, err := proto.Marshal(shadowReq) 669 if err != nil { 670 return nil, fmt.Errorf("failed to marshal forwarding shadow request: %w", err) 671 } 672 673 return snappy.Encode(buffer[:0], encoded), nil 674 } 675 676 // buildPseudoIDWithLabelsLikelySorted will build a pseudo ID that can be 677 // hashed/etc (but not used as primary key since not escaped), it expects the 678 // input labels to be likely sorted (so can avoid invoking sort in the regular 679 // case where series have labels already sorted when sent to remote write 680 // endpoint, which is commonly the case). 681 func buildPseudoIDWithLabelsLikelySorted( 682 labels []prompb.Label, 683 buffer []byte, 684 ) []byte { 685 for i, l := range labels { 686 if i > 0 && bytes.Compare(l.Name, labels[i-1].Name) < 0 { 687 // Sort. 688 sort.Sort(sortableLabels(labels)) 689 // Rebuild. 690 return buildPseudoIDWithLabelsLikelySorted(labels, buffer[:0]) 691 } 692 buffer = append(buffer, l.Name...) 693 buffer = append(buffer, '=') 694 buffer = append(buffer, l.Value...) 695 if i < len(labels)-1 { 696 buffer = append(buffer, ',') 697 } 698 } 699 return buffer 700 } 701 702 func (h *PromWriteHandler) maybeLogLabelsWithTooLongLiterals(logger *zap.Logger, label prompb.Label) { 703 if atomic.AddUint32(&h.numLiteralIsTooLong, 1) > maxLiteralIsTooLongLogCount { 704 return 705 } 706 707 safePrefix := func(b []byte, l int) []byte { 708 if len(b) <= l { 709 return b 710 } 711 return b[:l] 712 } 713 714 logger.Warn("label exceeds literal length limits", 715 zap.String("namePrefix", string(safePrefix(label.Name, literalPrefixLength))), 716 zap.Int("nameLength", len(label.Name)), 717 zap.String("valuePrefix", string(safePrefix(label.Value, literalPrefixLength))), 718 zap.Int("valueLength", len(label.Value)), 719 ) 720 } 721 722 func newPromTSIter( 723 timeseries []prompb.TimeSeries, 724 tagOpts models.TagOptions, 725 storeMetricsType bool, 726 ) (*promTSIter, error) { 727 // Construct the tags and datapoints upfront so that if the iterator 728 // is reset, we don't have to generate them twice. 729 var ( 730 tags = make([]models.Tags, 0, len(timeseries)) 731 datapoints = make([]ts.Datapoints, 0, len(timeseries)) 732 seriesAttributes = make([]ts.SeriesAttributes, 0, len(timeseries)) 733 ) 734 735 graphiteTagOpts := tagOpts.SetIDSchemeType(models.TypeGraphite) 736 for _, promTS := range timeseries { 737 attributes, err := storage.PromTimeSeriesToSeriesAttributes(promTS) 738 if err != nil { 739 return nil, err 740 } 741 742 // Set the tag options based on the incoming source. 743 opts := tagOpts 744 if attributes.Source == ts.SourceTypeGraphite { 745 opts = graphiteTagOpts 746 } 747 748 seriesAttributes = append(seriesAttributes, attributes) 749 tags = append(tags, storage.PromLabelsToM3Tags(promTS.Labels, opts)) 750 datapoints = append(datapoints, storage.PromSamplesToM3Datapoints(promTS.Samples)) 751 } 752 753 return &promTSIter{ 754 attributes: seriesAttributes, 755 idx: -1, 756 tags: tags, 757 datapoints: datapoints, 758 storeMetricsType: storeMetricsType, 759 }, nil 760 } 761 762 type promTSIter struct { 763 idx int 764 err error 765 attributes []ts.SeriesAttributes 766 tags []models.Tags 767 datapoints []ts.Datapoints 768 metadatas []ts.Metadata 769 annotation []byte 770 771 storeMetricsType bool 772 } 773 774 func (i *promTSIter) Next() bool { 775 if i.err != nil { 776 return false 777 } 778 779 i.idx++ 780 if i.idx >= len(i.tags) { 781 return false 782 } 783 784 if !i.storeMetricsType { 785 return true 786 } 787 788 annotationPayload, err := storage.SeriesAttributesToAnnotationPayload(i.attributes[i.idx]) 789 if err != nil { 790 i.err = err 791 return false 792 } 793 794 i.annotation, err = annotationPayload.Marshal() 795 if err != nil { 796 i.err = err 797 return false 798 } 799 800 if len(i.annotation) == 0 { 801 i.annotation = nil 802 } 803 804 return true 805 } 806 807 func (i *promTSIter) Current() ingest.IterValue { 808 if len(i.tags) == 0 || i.idx < 0 || i.idx >= len(i.tags) { 809 return defaultValue 810 } 811 812 value := ingest.IterValue{ 813 Tags: i.tags[i.idx], 814 Datapoints: i.datapoints[i.idx], 815 Attributes: i.attributes[i.idx], 816 Unit: xtime.Millisecond, 817 Annotation: i.annotation, 818 } 819 if i.idx < len(i.metadatas) { 820 value.Metadata = i.metadatas[i.idx] 821 } 822 return value 823 } 824 825 func (i *promTSIter) Reset() error { 826 i.idx = -1 827 i.err = nil 828 i.annotation = nil 829 830 return nil 831 } 832 833 func (i *promTSIter) Error() error { 834 return i.err 835 } 836 837 func (i *promTSIter) SetCurrentMetadata(metadata ts.Metadata) { 838 if len(i.metadatas) == 0 { 839 i.metadatas = make([]ts.Metadata, len(i.tags)) 840 } 841 if i.idx < 0 || i.idx >= len(i.metadatas) { 842 return 843 } 844 i.metadatas[i.idx] = metadata 845 } 846 847 type sortableLabels []prompb.Label 848 849 func (t sortableLabels) Len() int { return len(t) } 850 func (t sortableLabels) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 851 func (t sortableLabels) Less(i, j int) bool { 852 return bytes.Compare(t[i].Name, t[j].Name) == -1 853 }