github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/distributor/query.go (about) 1 package distributor 2 3 import ( 4 "context" 5 "io" 6 "sort" 7 "time" 8 9 "github.com/grafana/dskit/grpcutil" 10 "github.com/grafana/dskit/ring" 11 "github.com/opentracing/opentracing-go" 12 "github.com/prometheus/common/model" 13 "github.com/prometheus/prometheus/pkg/labels" 14 "github.com/weaveworks/common/instrument" 15 16 "github.com/cortexproject/cortex/pkg/cortexpb" 17 ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" 18 "github.com/cortexproject/cortex/pkg/querier/stats" 19 "github.com/cortexproject/cortex/pkg/tenant" 20 "github.com/cortexproject/cortex/pkg/util" 21 "github.com/cortexproject/cortex/pkg/util/extract" 22 "github.com/cortexproject/cortex/pkg/util/limiter" 23 "github.com/cortexproject/cortex/pkg/util/validation" 24 ) 25 26 // Query multiple ingesters and returns a Matrix of samples. 27 func (d *Distributor) Query(ctx context.Context, from, to model.Time, matchers ...*labels.Matcher) (model.Matrix, error) { 28 var matrix model.Matrix 29 err := instrument.CollectedRequest(ctx, "Distributor.Query", d.queryDuration, instrument.ErrorCode, func(ctx context.Context) error { 30 req, err := ingester_client.ToQueryRequest(from, to, matchers) 31 if err != nil { 32 return err 33 } 34 35 replicationSet, err := d.GetIngestersForQuery(ctx, matchers...) 36 if err != nil { 37 return err 38 } 39 40 matrix, err = d.queryIngesters(ctx, replicationSet, req) 41 if err != nil { 42 return err 43 } 44 45 if s := opentracing.SpanFromContext(ctx); s != nil { 46 s.LogKV("series", len(matrix)) 47 } 48 return nil 49 }) 50 return matrix, err 51 } 52 53 func (d *Distributor) QueryExemplars(ctx context.Context, from, to model.Time, matchers ...[]*labels.Matcher) (*ingester_client.ExemplarQueryResponse, error) { 54 var result *ingester_client.ExemplarQueryResponse 55 err := instrument.CollectedRequest(ctx, "Distributor.QueryExemplars", d.queryDuration, instrument.ErrorCode, func(ctx context.Context) error { 56 req, err := ingester_client.ToExemplarQueryRequest(from, to, matchers...) 57 if err != nil { 58 return err 59 } 60 61 // We ask for all ingesters without passing matchers because exemplar queries take in an array of array of label matchers. 62 replicationSet, err := d.GetIngestersForQuery(ctx) 63 if err != nil { 64 return err 65 } 66 67 result, err = d.queryIngestersExemplars(ctx, replicationSet, req) 68 if err != nil { 69 return err 70 } 71 72 if s := opentracing.SpanFromContext(ctx); s != nil { 73 s.LogKV("series", len(result.Timeseries)) 74 } 75 return nil 76 }) 77 return result, err 78 } 79 80 // QueryStream multiple ingesters via the streaming interface and returns big ol' set of chunks. 81 func (d *Distributor) QueryStream(ctx context.Context, from, to model.Time, matchers ...*labels.Matcher) (*ingester_client.QueryStreamResponse, error) { 82 var result *ingester_client.QueryStreamResponse 83 err := instrument.CollectedRequest(ctx, "Distributor.QueryStream", d.queryDuration, instrument.ErrorCode, func(ctx context.Context) error { 84 req, err := ingester_client.ToQueryRequest(from, to, matchers) 85 if err != nil { 86 return err 87 } 88 89 replicationSet, err := d.GetIngestersForQuery(ctx, matchers...) 90 if err != nil { 91 return err 92 } 93 94 result, err = d.queryIngesterStream(ctx, replicationSet, req) 95 if err != nil { 96 return err 97 } 98 99 if s := opentracing.SpanFromContext(ctx); s != nil { 100 s.LogKV("chunk-series", len(result.GetChunkseries()), "time-series", len(result.GetTimeseries())) 101 } 102 return nil 103 }) 104 return result, err 105 } 106 107 // GetIngestersForQuery returns a replication set including all ingesters that should be queried 108 // to fetch series matching input label matchers. 109 func (d *Distributor) GetIngestersForQuery(ctx context.Context, matchers ...*labels.Matcher) (ring.ReplicationSet, error) { 110 userID, err := tenant.TenantID(ctx) 111 if err != nil { 112 return ring.ReplicationSet{}, err 113 } 114 115 // If shuffle sharding is enabled we should only query ingesters which are 116 // part of the tenant's subring. 117 if d.cfg.ShardingStrategy == util.ShardingStrategyShuffle { 118 shardSize := d.limits.IngestionTenantShardSize(userID) 119 lookbackPeriod := d.cfg.ShuffleShardingLookbackPeriod 120 121 if shardSize > 0 && lookbackPeriod > 0 { 122 return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetReplicationSetForOperation(ring.Read) 123 } 124 } 125 126 // If "shard by all labels" is disabled, we can get ingesters by metricName if exists. 127 if !d.cfg.ShardByAllLabels && len(matchers) > 0 { 128 metricNameMatcher, _, ok := extract.MetricNameMatcherFromMatchers(matchers) 129 130 if ok && metricNameMatcher.Type == labels.MatchEqual { 131 return d.ingestersRing.Get(shardByMetricName(userID, metricNameMatcher.Value), ring.Read, nil, nil, nil) 132 } 133 } 134 135 return d.ingestersRing.GetReplicationSetForOperation(ring.Read) 136 } 137 138 // GetIngestersForMetadata returns a replication set including all ingesters that should be queried 139 // to fetch metadata (eg. label names/values or series). 140 func (d *Distributor) GetIngestersForMetadata(ctx context.Context) (ring.ReplicationSet, error) { 141 userID, err := tenant.TenantID(ctx) 142 if err != nil { 143 return ring.ReplicationSet{}, err 144 } 145 146 // If shuffle sharding is enabled we should only query ingesters which are 147 // part of the tenant's subring. 148 if d.cfg.ShardingStrategy == util.ShardingStrategyShuffle { 149 shardSize := d.limits.IngestionTenantShardSize(userID) 150 lookbackPeriod := d.cfg.ShuffleShardingLookbackPeriod 151 152 if shardSize > 0 && lookbackPeriod > 0 { 153 return d.ingestersRing.ShuffleShardWithLookback(userID, shardSize, lookbackPeriod, time.Now()).GetReplicationSetForOperation(ring.Read) 154 } 155 } 156 157 return d.ingestersRing.GetReplicationSetForOperation(ring.Read) 158 } 159 160 // queryIngesters queries the ingesters via the older, sample-based API. 161 func (d *Distributor) queryIngesters(ctx context.Context, replicationSet ring.ReplicationSet, req *ingester_client.QueryRequest) (model.Matrix, error) { 162 // Fetch samples from multiple ingesters in parallel, using the replicationSet 163 // to deal with consistency. 164 results, err := replicationSet.Do(ctx, d.cfg.ExtraQueryDelay, func(ctx context.Context, ing *ring.InstanceDesc) (interface{}, error) { 165 client, err := d.ingesterPool.GetClientFor(ing.Addr) 166 if err != nil { 167 return nil, err 168 } 169 170 resp, err := client.(ingester_client.IngesterClient).Query(ctx, req) 171 d.ingesterQueries.WithLabelValues(ing.Addr).Inc() 172 if err != nil { 173 d.ingesterQueryFailures.WithLabelValues(ing.Addr).Inc() 174 return nil, err 175 } 176 177 return ingester_client.FromQueryResponse(resp), nil 178 }) 179 if err != nil { 180 return nil, err 181 } 182 183 // Merge the results into a single matrix. 184 fpToSampleStream := map[model.Fingerprint]*model.SampleStream{} 185 for _, result := range results { 186 for _, ss := range result.(model.Matrix) { 187 fp := ss.Metric.Fingerprint() 188 mss, ok := fpToSampleStream[fp] 189 if !ok { 190 mss = &model.SampleStream{ 191 Metric: ss.Metric, 192 } 193 fpToSampleStream[fp] = mss 194 } 195 mss.Values = util.MergeSampleSets(mss.Values, ss.Values) 196 } 197 } 198 result := model.Matrix{} 199 for _, ss := range fpToSampleStream { 200 result = append(result, ss) 201 } 202 203 return result, nil 204 } 205 206 // mergeExemplarSets merges and dedupes two sets of already sorted exemplar pairs. 207 // Both a and b should be lists of exemplars from the same series. 208 // Defined here instead of pkg/util to avoid a import cycle. 209 func mergeExemplarSets(a, b []cortexpb.Exemplar) []cortexpb.Exemplar { 210 result := make([]cortexpb.Exemplar, 0, len(a)+len(b)) 211 i, j := 0, 0 212 for i < len(a) && j < len(b) { 213 if a[i].TimestampMs < b[j].TimestampMs { 214 result = append(result, a[i]) 215 i++ 216 } else if a[i].TimestampMs > b[j].TimestampMs { 217 result = append(result, b[j]) 218 j++ 219 } else { 220 result = append(result, a[i]) 221 i++ 222 j++ 223 } 224 } 225 // Add the rest of a or b. One of them is empty now. 226 result = append(result, a[i:]...) 227 result = append(result, b[j:]...) 228 return result 229 } 230 231 // queryIngestersExemplars queries the ingesters for exemplars. 232 func (d *Distributor) queryIngestersExemplars(ctx context.Context, replicationSet ring.ReplicationSet, req *ingester_client.ExemplarQueryRequest) (*ingester_client.ExemplarQueryResponse, error) { 233 // Fetch exemplars from multiple ingesters in parallel, using the replicationSet 234 // to deal with consistency. 235 results, err := replicationSet.Do(ctx, d.cfg.ExtraQueryDelay, func(ctx context.Context, ing *ring.InstanceDesc) (interface{}, error) { 236 client, err := d.ingesterPool.GetClientFor(ing.Addr) 237 if err != nil { 238 return nil, err 239 } 240 241 resp, err := client.(ingester_client.IngesterClient).QueryExemplars(ctx, req) 242 d.ingesterQueries.WithLabelValues(ing.Addr).Inc() 243 if err != nil { 244 d.ingesterQueryFailures.WithLabelValues(ing.Addr).Inc() 245 return nil, err 246 } 247 248 return resp, nil 249 }) 250 if err != nil { 251 return nil, err 252 } 253 254 // Merge results from replication set. 255 var keys []string 256 exemplarResults := make(map[string]cortexpb.TimeSeries) 257 for _, result := range results { 258 r := result.(*ingester_client.ExemplarQueryResponse) 259 for _, ts := range r.Timeseries { 260 lbls := cortexpb.FromLabelAdaptersToLabels(ts.Labels).String() 261 e, ok := exemplarResults[lbls] 262 if !ok { 263 exemplarResults[lbls] = ts 264 keys = append(keys, lbls) 265 } 266 // Merge in any missing values from another ingesters exemplars for this series. 267 e.Exemplars = mergeExemplarSets(e.Exemplars, ts.Exemplars) 268 } 269 } 270 271 // Query results from each ingester were sorted, but are not necessarily still sorted after merging. 272 sort.Strings(keys) 273 274 result := make([]cortexpb.TimeSeries, len(exemplarResults)) 275 for i, k := range keys { 276 result[i] = exemplarResults[k] 277 } 278 279 return &ingester_client.ExemplarQueryResponse{Timeseries: result}, nil 280 } 281 282 // queryIngesterStream queries the ingesters using the new streaming API. 283 func (d *Distributor) queryIngesterStream(ctx context.Context, replicationSet ring.ReplicationSet, req *ingester_client.QueryRequest) (*ingester_client.QueryStreamResponse, error) { 284 var ( 285 queryLimiter = limiter.QueryLimiterFromContextWithFallback(ctx) 286 reqStats = stats.FromContext(ctx) 287 ) 288 289 // Fetch samples from multiple ingesters 290 results, err := replicationSet.Do(ctx, d.cfg.ExtraQueryDelay, func(ctx context.Context, ing *ring.InstanceDesc) (interface{}, error) { 291 client, err := d.ingesterPool.GetClientFor(ing.Addr) 292 if err != nil { 293 return nil, err 294 } 295 d.ingesterQueries.WithLabelValues(ing.Addr).Inc() 296 297 stream, err := client.(ingester_client.IngesterClient).QueryStream(ctx, req) 298 if err != nil { 299 d.ingesterQueryFailures.WithLabelValues(ing.Addr).Inc() 300 return nil, err 301 } 302 defer stream.CloseSend() //nolint:errcheck 303 304 result := &ingester_client.QueryStreamResponse{} 305 for { 306 resp, err := stream.Recv() 307 if err == io.EOF { 308 break 309 } else if err != nil { 310 // Do not track a failure if the context was canceled. 311 if !grpcutil.IsGRPCContextCanceled(err) { 312 d.ingesterQueryFailures.WithLabelValues(ing.Addr).Inc() 313 } 314 315 return nil, err 316 } 317 318 // Enforce the max chunks limits. 319 if chunkLimitErr := queryLimiter.AddChunks(resp.ChunksCount()); chunkLimitErr != nil { 320 return nil, validation.LimitError(chunkLimitErr.Error()) 321 } 322 323 for _, series := range resp.Chunkseries { 324 if limitErr := queryLimiter.AddSeries(series.Labels); limitErr != nil { 325 return nil, validation.LimitError(limitErr.Error()) 326 } 327 } 328 329 if chunkBytesLimitErr := queryLimiter.AddChunkBytes(resp.ChunksSize()); chunkBytesLimitErr != nil { 330 return nil, validation.LimitError(chunkBytesLimitErr.Error()) 331 } 332 333 for _, series := range resp.Timeseries { 334 if limitErr := queryLimiter.AddSeries(series.Labels); limitErr != nil { 335 return nil, validation.LimitError(limitErr.Error()) 336 } 337 } 338 339 result.Chunkseries = append(result.Chunkseries, resp.Chunkseries...) 340 result.Timeseries = append(result.Timeseries, resp.Timeseries...) 341 } 342 return result, nil 343 }) 344 if err != nil { 345 return nil, err 346 } 347 348 hashToChunkseries := map[string]ingester_client.TimeSeriesChunk{} 349 hashToTimeSeries := map[string]cortexpb.TimeSeries{} 350 351 for _, result := range results { 352 response := result.(*ingester_client.QueryStreamResponse) 353 354 // Parse any chunk series 355 for _, series := range response.Chunkseries { 356 key := ingester_client.LabelsToKeyString(cortexpb.FromLabelAdaptersToLabels(series.Labels)) 357 existing := hashToChunkseries[key] 358 existing.Labels = series.Labels 359 existing.Chunks = append(existing.Chunks, series.Chunks...) 360 hashToChunkseries[key] = existing 361 } 362 363 // Parse any time series 364 for _, series := range response.Timeseries { 365 key := ingester_client.LabelsToKeyString(cortexpb.FromLabelAdaptersToLabels(series.Labels)) 366 existing := hashToTimeSeries[key] 367 existing.Labels = series.Labels 368 if existing.Samples == nil { 369 existing.Samples = series.Samples 370 } else { 371 existing.Samples = mergeSamples(existing.Samples, series.Samples) 372 } 373 hashToTimeSeries[key] = existing 374 } 375 } 376 377 resp := &ingester_client.QueryStreamResponse{ 378 Chunkseries: make([]ingester_client.TimeSeriesChunk, 0, len(hashToChunkseries)), 379 Timeseries: make([]cortexpb.TimeSeries, 0, len(hashToTimeSeries)), 380 } 381 for _, series := range hashToChunkseries { 382 resp.Chunkseries = append(resp.Chunkseries, series) 383 } 384 for _, series := range hashToTimeSeries { 385 resp.Timeseries = append(resp.Timeseries, series) 386 } 387 388 reqStats.AddFetchedSeries(uint64(len(resp.Chunkseries) + len(resp.Timeseries))) 389 reqStats.AddFetchedChunkBytes(uint64(resp.ChunksSize())) 390 391 return resp, nil 392 } 393 394 // Merges and dedupes two sorted slices with samples together. 395 func mergeSamples(a, b []cortexpb.Sample) []cortexpb.Sample { 396 if sameSamples(a, b) { 397 return a 398 } 399 400 result := make([]cortexpb.Sample, 0, len(a)+len(b)) 401 i, j := 0, 0 402 for i < len(a) && j < len(b) { 403 if a[i].TimestampMs < b[j].TimestampMs { 404 result = append(result, a[i]) 405 i++ 406 } else if a[i].TimestampMs > b[j].TimestampMs { 407 result = append(result, b[j]) 408 j++ 409 } else { 410 result = append(result, a[i]) 411 i++ 412 j++ 413 } 414 } 415 // Add the rest of a or b. One of them is empty now. 416 result = append(result, a[i:]...) 417 result = append(result, b[j:]...) 418 return result 419 } 420 421 func sameSamples(a, b []cortexpb.Sample) bool { 422 if len(a) != len(b) { 423 return false 424 } 425 426 for i := 0; i < len(a); i++ { 427 if a[i] != b[i] { 428 return false 429 } 430 } 431 return true 432 }