github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/batch.go (about) 1 package storage 2 3 import ( 4 "context" 5 "sort" 6 "time" 7 8 "github.com/go-kit/log/level" 9 "github.com/pkg/errors" 10 "github.com/prometheus/client_golang/prometheus" 11 "github.com/prometheus/client_golang/prometheus/promauto" 12 "github.com/prometheus/common/model" 13 "github.com/prometheus/prometheus/model/labels" 14 "github.com/prometheus/prometheus/promql" 15 16 "github.com/grafana/loki/pkg/chunkenc" 17 "github.com/grafana/loki/pkg/iter" 18 "github.com/grafana/loki/pkg/logproto" 19 "github.com/grafana/loki/pkg/logql/log" 20 "github.com/grafana/loki/pkg/logql/syntax" 21 "github.com/grafana/loki/pkg/logqlmodel/stats" 22 "github.com/grafana/loki/pkg/querier/astmapper" 23 "github.com/grafana/loki/pkg/storage/chunk" 24 "github.com/grafana/loki/pkg/storage/chunk/fetcher" 25 "github.com/grafana/loki/pkg/storage/config" 26 util_log "github.com/grafana/loki/pkg/util/log" 27 ) 28 29 type ChunkMetrics struct { 30 refs *prometheus.CounterVec 31 series *prometheus.CounterVec 32 chunks *prometheus.CounterVec 33 batches *prometheus.HistogramVec 34 } 35 36 const ( 37 statusDiscarded = "discarded" 38 statusMatched = "matched" 39 ) 40 41 func NewChunkMetrics(r prometheus.Registerer, maxBatchSize int) *ChunkMetrics { 42 buckets := 5 43 if maxBatchSize < buckets { 44 maxBatchSize = buckets 45 } 46 47 return &ChunkMetrics{ 48 refs: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 49 Namespace: "loki", 50 Subsystem: "index", 51 Name: "chunk_refs_total", 52 Help: "Number of chunks refs downloaded, partitioned by whether they intersect the query bounds.", 53 }, []string{"status"}), 54 series: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 55 Namespace: "loki", 56 Subsystem: "store", 57 Name: "series_total", 58 Help: "Number of series referenced by a query, partitioned by whether they satisfy matchers.", 59 }, []string{"status"}), 60 chunks: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 61 Namespace: "loki", 62 Subsystem: "store", 63 Name: "chunks_downloaded_total", 64 Help: "Number of chunks referenced or downloaded, partitioned by if they satisfy matchers.", 65 }, []string{"status"}), 66 batches: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ 67 Namespace: "loki", 68 Subsystem: "store", 69 Name: "chunks_per_batch", 70 Help: "The chunk batch size, partitioned by if they satisfy matchers.", 71 72 // split buckets evenly across 0->maxBatchSize 73 Buckets: prometheus.LinearBuckets(0, float64(maxBatchSize/buckets), buckets+1), // increment buckets by one to ensure upper bound bucket exists. 74 }, []string{"status"}), 75 } 76 } 77 78 // batchChunkIterator iterates through chunks by batch of `batchSize`. 79 // Since chunks can overlap across batches for each iteration the iterator will keep all overlapping 80 // chunks with the next chunk from the next batch and added it to the next iteration. In this case the boundaries of the batch 81 // is reduced to non-overlapping chunks boundaries. 82 type batchChunkIterator struct { 83 schemas config.SchemaConfig 84 chunks lazyChunks 85 batchSize int 86 lastOverlapping []*LazyChunk 87 metrics *ChunkMetrics 88 matchers []*labels.Matcher 89 chunkFilterer chunk.Filterer 90 91 begun bool 92 ctx context.Context 93 start, end time.Time 94 direction logproto.Direction 95 next chan *chunkBatch 96 } 97 98 // newBatchChunkIterator creates a new batch iterator with the given batchSize. 99 func newBatchChunkIterator( 100 ctx context.Context, 101 s config.SchemaConfig, 102 chunks []*LazyChunk, 103 batchSize int, 104 direction logproto.Direction, 105 start, end time.Time, 106 metrics *ChunkMetrics, 107 matchers []*labels.Matcher, 108 chunkFilterer chunk.Filterer, 109 ) *batchChunkIterator { 110 // __name__ is not something we filter by because it's a constant in loki 111 // and only used for upstream compatibility; therefore remove it. 112 // The same applies to the sharding label which is injected by the cortex storage code. 113 matchers = removeMatchersByName(matchers, labels.MetricName, astmapper.ShardLabel) 114 res := &batchChunkIterator{ 115 batchSize: batchSize, 116 schemas: s, 117 metrics: metrics, 118 matchers: matchers, 119 start: start, 120 end: end, 121 direction: direction, 122 ctx: ctx, 123 chunks: lazyChunks{direction: direction, chunks: chunks}, 124 next: make(chan *chunkBatch), 125 chunkFilterer: chunkFilterer, 126 } 127 sort.Sort(res.chunks) 128 return res 129 } 130 131 // Start is idempotent and will begin the processing thread which seeds the iterator data. 132 func (it *batchChunkIterator) Start() { 133 if !it.begun { 134 it.begun = true 135 go it.loop() 136 } 137 } 138 139 func (it *batchChunkIterator) loop() { 140 for { 141 if it.chunks.Len() == 0 { 142 close(it.next) 143 return 144 } 145 select { 146 case <-it.ctx.Done(): 147 close(it.next) 148 return 149 case it.next <- it.nextBatch(): 150 } 151 } 152 } 153 154 func (it *batchChunkIterator) Next() *chunkBatch { 155 it.Start() // Ensure the iterator has started. 156 return <-it.next 157 } 158 159 func (it *batchChunkIterator) nextBatch() (res *chunkBatch) { 160 defer func() { 161 if p := recover(); p != nil { 162 level.Error(util_log.Logger).Log("msg", "panic while fetching chunks", "panic", p) 163 res = &chunkBatch{ 164 err: errors.Errorf("panic while fecthing chunks %+v", p), 165 } 166 } 167 }() 168 // the first chunk of the batch 169 headChunk := it.chunks.Peek() 170 from, through := it.start, it.end 171 batch := make([]*LazyChunk, 0, it.batchSize+len(it.lastOverlapping)) 172 var nextChunk *LazyChunk 173 174 var includesOverlap bool 175 176 for it.chunks.Len() > 0 { 177 178 // pop the next batch of chunks and append/prepend previous overlapping chunks 179 // so we can merge/de-dupe overlapping entries. 180 if !includesOverlap && it.direction == logproto.FORWARD { 181 batch = append(batch, it.lastOverlapping...) 182 } 183 batch = append(batch, it.chunks.pop(it.batchSize)...) 184 if !includesOverlap && it.direction == logproto.BACKWARD { 185 batch = append(batch, it.lastOverlapping...) 186 } 187 188 includesOverlap = true 189 190 if it.chunks.Len() > 0 { 191 nextChunk = it.chunks.Peek() 192 // we max out our iterator boundaries to the next chunks in the queue 193 // so that overlapping chunks are together 194 if it.direction == logproto.BACKWARD { 195 from = time.Unix(0, nextChunk.Chunk.Through.UnixNano()) 196 197 // we have to reverse the inclusivity of the chunk iterator from 198 // [from, through) to (from, through] for backward queries, except when 199 // the batch's `from` is equal to the query's Start. This can be achieved 200 // by shifting `from` by one nanosecond. 201 if !from.Equal(it.start) { 202 from = from.Add(time.Nanosecond) 203 } 204 } else { 205 through = time.Unix(0, nextChunk.Chunk.From.UnixNano()) 206 } 207 // we save all overlapping chunks as they are also needed in the next batch to properly order entries. 208 // If we have chunks like below: 209 // ┌──────────────┐ 210 // │ # 47 │ 211 // └──────────────┘ 212 // ┌──────────────────────────┐ 213 // │ # 48 | 214 // └──────────────────────────┘ 215 // ┌──────────────┐ 216 // │ # 49 │ 217 // └──────────────┘ 218 // ┌────────────────────┐ 219 // │ # 50 │ 220 // └────────────────────┘ 221 // 222 // And nextChunk is # 49, we need to keep references to #47 and #48 as they won't be 223 // iterated over completely (we're clipping through to #49's from) and then add them to the next batch. 224 225 } 226 227 if it.direction == logproto.BACKWARD { 228 through = time.Unix(0, headChunk.Chunk.Through.UnixNano()) 229 230 if through.After(it.end) { 231 through = it.end 232 } 233 234 // we have to reverse the inclusivity of the chunk iterator from 235 // [from, through) to (from, through] for backward queries, except when 236 // the batch's `through` is equal to the query's End. This can be achieved 237 // by shifting `through` by one nanosecond. 238 if !through.Equal(it.end) { 239 through = through.Add(time.Nanosecond) 240 } 241 } else { 242 243 from = time.Unix(0, headChunk.Chunk.From.UnixNano()) 244 245 // if the start of the batch is equal to the end of the query, since the end is not inclusive we can discard that batch. 246 if from.Equal(it.end) { 247 if it.end != it.start { 248 return nil 249 } 250 // unless end and start are equal in which case start and end are both inclusive. 251 from = it.end 252 } 253 // when clipping the from it should never be before the start. 254 // Doing so would include entries not requested. 255 if from.Before(it.start) { 256 from = it.start 257 } 258 } 259 260 // it's possible that the current batch and the next batch are fully overlapping in which case 261 // we should keep adding more items until the batch boundaries difference is positive. 262 if through.Sub(from) > 0 { 263 break 264 } 265 } 266 267 // If every chunk overlaps and we exhaust fetching chunks before ever finding a non overlapping chunk 268 // in this case it will be possible to have a through value which is older or equal to our from value 269 // If that happens we reset the bounds according to the iteration direction 270 if through.Sub(from) <= 0 { 271 if it.direction == logproto.BACKWARD { 272 from = it.start 273 } else { 274 through = it.end 275 } 276 } 277 278 if it.chunks.Len() > 0 { 279 it.lastOverlapping = it.lastOverlapping[:0] 280 for _, c := range batch { 281 if c.IsOverlapping(nextChunk, it.direction) { 282 it.lastOverlapping = append(it.lastOverlapping, c) 283 } 284 } 285 } 286 // download chunk for this batch. 287 chksBySeries, err := fetchChunkBySeries(it.ctx, it.schemas, it.metrics, batch, it.matchers, it.chunkFilterer) 288 if err != nil { 289 return &chunkBatch{err: err} 290 } 291 return &chunkBatch{ 292 chunksBySeries: chksBySeries, 293 err: err, 294 from: from, 295 through: through, 296 nextChunk: nextChunk, 297 } 298 } 299 300 type chunkBatch struct { 301 chunksBySeries map[model.Fingerprint][][]*LazyChunk 302 err error 303 304 from, through time.Time 305 nextChunk *LazyChunk 306 } 307 308 type logBatchIterator struct { 309 *batchChunkIterator 310 curr iter.EntryIterator 311 err error 312 313 ctx context.Context 314 cancel context.CancelFunc 315 pipeline syntax.Pipeline 316 } 317 318 func newLogBatchIterator( 319 ctx context.Context, 320 schemas config.SchemaConfig, 321 metrics *ChunkMetrics, 322 chunks []*LazyChunk, 323 batchSize int, 324 matchers []*labels.Matcher, 325 pipeline syntax.Pipeline, 326 direction logproto.Direction, 327 start, end time.Time, 328 chunkFilterer chunk.Filterer, 329 ) (iter.EntryIterator, error) { 330 ctx, cancel := context.WithCancel(ctx) 331 return &logBatchIterator{ 332 pipeline: pipeline, 333 ctx: ctx, 334 cancel: cancel, 335 batchChunkIterator: newBatchChunkIterator(ctx, schemas, chunks, batchSize, direction, start, end, metrics, matchers, chunkFilterer), 336 }, nil 337 } 338 339 func (it *logBatchIterator) Labels() string { 340 return it.curr.Labels() 341 } 342 343 func (it *logBatchIterator) StreamHash() uint64 { 344 return it.curr.StreamHash() 345 } 346 347 func (it *logBatchIterator) Error() error { 348 if it.err != nil { 349 return it.err 350 } 351 if it.curr != nil && it.curr.Error() != nil { 352 return it.curr.Error() 353 } 354 if it.ctx.Err() != nil { 355 return it.ctx.Err() 356 } 357 return nil 358 } 359 360 func (it *logBatchIterator) Close() error { 361 it.cancel() 362 if it.curr != nil { 363 return it.curr.Close() 364 } 365 return nil 366 } 367 368 func (it *logBatchIterator) Entry() logproto.Entry { 369 return it.curr.Entry() 370 } 371 372 func (it *logBatchIterator) Next() bool { 373 // for loop to avoid recursion 374 for { 375 if it.curr != nil && it.curr.Next() { 376 return true 377 } 378 // close previous iterator 379 if it.curr != nil { 380 it.err = it.curr.Close() 381 } 382 next := it.batchChunkIterator.Next() 383 if next == nil { 384 return false 385 } 386 if next.err != nil { 387 it.err = next.err 388 return false 389 } 390 var err error 391 it.curr, err = it.newChunksIterator(next) 392 if err != nil { 393 it.err = err 394 return false 395 } 396 } 397 } 398 399 // newChunksIterator creates an iterator over a set of lazychunks. 400 func (it *logBatchIterator) newChunksIterator(b *chunkBatch) (iter.EntryIterator, error) { 401 iters, err := it.buildIterators(b.chunksBySeries, b.from, b.through, b.nextChunk) 402 if err != nil { 403 return nil, err 404 } 405 if len(iters) == 1 { 406 return iters[0], nil 407 } 408 return iter.NewSortEntryIterator(iters, it.direction), nil 409 } 410 411 func (it *logBatchIterator) buildIterators(chks map[model.Fingerprint][][]*LazyChunk, from, through time.Time, nextChunk *LazyChunk) ([]iter.EntryIterator, error) { 412 result := make([]iter.EntryIterator, 0, len(chks)) 413 for _, chunks := range chks { 414 if len(chunks) != 0 && len(chunks[0]) != 0 { 415 streamPipeline := it.pipeline.ForStream(labels.NewBuilder(chunks[0][0].Chunk.Metric).Del(labels.MetricName).Labels()) 416 iterator, err := it.buildHeapIterator(chunks, from, through, streamPipeline, nextChunk) 417 if err != nil { 418 return nil, err 419 } 420 421 result = append(result, iterator) 422 } 423 } 424 425 return result, nil 426 } 427 428 func (it *logBatchIterator) buildHeapIterator(chks [][]*LazyChunk, from, through time.Time, streamPipeline log.StreamPipeline, nextChunk *LazyChunk) (iter.EntryIterator, error) { 429 result := make([]iter.EntryIterator, 0, len(chks)) 430 431 for i := range chks { 432 iterators := make([]iter.EntryIterator, 0, len(chks[i])) 433 for j := range chks[i] { 434 if !chks[i][j].IsValid { 435 continue 436 } 437 iterator, err := chks[i][j].Iterator(it.ctx, from, through, it.direction, streamPipeline, nextChunk) 438 if err != nil { 439 return nil, err 440 } 441 iterators = append(iterators, iterator) 442 } 443 if it.direction == logproto.BACKWARD { 444 for i, j := 0, len(iterators)-1; i < j; i, j = i+1, j-1 { 445 iterators[i], iterators[j] = iterators[j], iterators[i] 446 } 447 } 448 result = append(result, iter.NewNonOverlappingIterator(iterators)) 449 } 450 451 return iter.NewMergeEntryIterator(it.ctx, result, it.direction), nil 452 } 453 454 type sampleBatchIterator struct { 455 *batchChunkIterator 456 curr iter.SampleIterator 457 err error 458 459 ctx context.Context 460 cancel context.CancelFunc 461 extractor syntax.SampleExtractor 462 } 463 464 func newSampleBatchIterator( 465 ctx context.Context, 466 schemas config.SchemaConfig, 467 metrics *ChunkMetrics, 468 chunks []*LazyChunk, 469 batchSize int, 470 matchers []*labels.Matcher, 471 extractor syntax.SampleExtractor, 472 start, end time.Time, 473 chunkFilterer chunk.Filterer, 474 ) (iter.SampleIterator, error) { 475 ctx, cancel := context.WithCancel(ctx) 476 return &sampleBatchIterator{ 477 extractor: extractor, 478 ctx: ctx, 479 cancel: cancel, 480 batchChunkIterator: newBatchChunkIterator(ctx, schemas, chunks, batchSize, logproto.FORWARD, start, end, metrics, matchers, chunkFilterer), 481 }, nil 482 } 483 484 func (it *sampleBatchIterator) Labels() string { 485 return it.curr.Labels() 486 } 487 488 func (it *sampleBatchIterator) StreamHash() uint64 { 489 return it.curr.StreamHash() 490 } 491 492 func (it *sampleBatchIterator) Error() error { 493 if it.err != nil { 494 return it.err 495 } 496 if it.curr != nil && it.curr.Error() != nil { 497 return it.curr.Error() 498 } 499 if it.ctx.Err() != nil { 500 return it.ctx.Err() 501 } 502 return nil 503 } 504 505 func (it *sampleBatchIterator) Close() error { 506 it.cancel() 507 if it.curr != nil { 508 return it.curr.Close() 509 } 510 return nil 511 } 512 513 func (it *sampleBatchIterator) Sample() logproto.Sample { 514 return it.curr.Sample() 515 } 516 517 func (it *sampleBatchIterator) Next() bool { 518 // for loop to avoid recursion 519 for it.ctx.Err() == nil { 520 if it.curr != nil && it.curr.Next() { 521 return true 522 } 523 // close previous iterator 524 if it.curr != nil { 525 it.err = it.curr.Close() 526 } 527 next := it.batchChunkIterator.Next() 528 if next == nil { 529 return false 530 } 531 if next.err != nil { 532 it.err = next.err 533 return false 534 } 535 var err error 536 it.curr, err = it.newChunksIterator(next) 537 if err != nil { 538 it.err = err 539 return false 540 } 541 } 542 return false 543 } 544 545 // newChunksIterator creates an iterator over a set of lazychunks. 546 func (it *sampleBatchIterator) newChunksIterator(b *chunkBatch) (iter.SampleIterator, error) { 547 iters, err := it.buildIterators(b.chunksBySeries, b.from, b.through, b.nextChunk) 548 if err != nil { 549 return nil, err 550 } 551 552 return iter.NewSortSampleIterator(iters), nil 553 } 554 555 func (it *sampleBatchIterator) buildIterators(chks map[model.Fingerprint][][]*LazyChunk, from, through time.Time, nextChunk *LazyChunk) ([]iter.SampleIterator, error) { 556 result := make([]iter.SampleIterator, 0, len(chks)) 557 for _, chunks := range chks { 558 if len(chunks) != 0 && len(chunks[0]) != 0 { 559 streamExtractor := it.extractor.ForStream(labels.NewBuilder(chunks[0][0].Chunk.Metric).Del(labels.MetricName).Labels()) 560 iterator, err := it.buildHeapIterator(chunks, from, through, streamExtractor, nextChunk) 561 if err != nil { 562 return nil, err 563 } 564 result = append(result, iterator) 565 } 566 } 567 568 return result, nil 569 } 570 571 func (it *sampleBatchIterator) buildHeapIterator(chks [][]*LazyChunk, from, through time.Time, streamExtractor log.StreamSampleExtractor, nextChunk *LazyChunk) (iter.SampleIterator, error) { 572 result := make([]iter.SampleIterator, 0, len(chks)) 573 574 for i := range chks { 575 iterators := make([]iter.SampleIterator, 0, len(chks[i])) 576 for j := range chks[i] { 577 if !chks[i][j].IsValid { 578 continue 579 } 580 iterator, err := chks[i][j].SampleIterator(it.ctx, from, through, streamExtractor, nextChunk) 581 if err != nil { 582 return nil, err 583 } 584 iterators = append(iterators, iterator) 585 } 586 result = append(result, iter.NewNonOverlappingSampleIterator(iterators)) 587 } 588 589 return iter.NewMergeSampleIterator(it.ctx, result), nil 590 } 591 592 func removeMatchersByName(matchers []*labels.Matcher, names ...string) []*labels.Matcher { 593 for _, omit := range names { 594 for i := range matchers { 595 if matchers[i].Name == omit { 596 matchers = append(matchers[:i], matchers[i+1:]...) 597 break 598 } 599 } 600 } 601 return matchers 602 } 603 604 func fetchChunkBySeries( 605 ctx context.Context, 606 s config.SchemaConfig, 607 metrics *ChunkMetrics, 608 chunks []*LazyChunk, 609 matchers []*labels.Matcher, 610 chunkFilter chunk.Filterer, 611 ) (map[model.Fingerprint][][]*LazyChunk, error) { 612 chksBySeries := partitionBySeriesChunks(chunks) 613 614 // Make sure the initial chunks are loaded. This is not one chunk 615 // per series, but rather a chunk per non-overlapping iterator. 616 if err := loadFirstChunks(ctx, s, chksBySeries); err != nil { 617 return nil, err 618 } 619 620 // Now that we have the first chunk for each series loaded, 621 // we can proceed to filter the series that don't match. 622 chksBySeries = filterSeriesByMatchers(chksBySeries, matchers, chunkFilter, metrics) 623 624 var allChunks []*LazyChunk 625 for _, series := range chksBySeries { 626 for _, chunks := range series { 627 allChunks = append(allChunks, chunks...) 628 } 629 } 630 631 // Finally we load all chunks not already loaded 632 if err := fetchLazyChunks(ctx, s, allChunks); err != nil { 633 return nil, err 634 } 635 metrics.chunks.WithLabelValues(statusMatched).Add(float64(len(allChunks))) 636 metrics.series.WithLabelValues(statusMatched).Add(float64(len(chksBySeries))) 637 metrics.batches.WithLabelValues(statusMatched).Observe(float64(len(allChunks))) 638 metrics.batches.WithLabelValues(statusDiscarded).Observe(float64(len(chunks) - len(allChunks))) 639 640 return chksBySeries, nil 641 } 642 643 func filterSeriesByMatchers( 644 chks map[model.Fingerprint][][]*LazyChunk, 645 matchers []*labels.Matcher, 646 chunkFilterer chunk.Filterer, 647 metrics *ChunkMetrics, 648 ) map[model.Fingerprint][][]*LazyChunk { 649 var filteredSeries, filteredChks int 650 651 removeSeries := func(fp model.Fingerprint, chunks [][]*LazyChunk) { 652 delete(chks, fp) 653 filteredSeries++ 654 655 for _, grp := range chunks { 656 filteredChks += len(grp) 657 } 658 } 659 outer: 660 for fp, chunks := range chks { 661 for _, matcher := range matchers { 662 if !matcher.Matches(chunks[0][0].Chunk.Metric.Get(matcher.Name)) { 663 removeSeries(fp, chunks) 664 continue outer 665 } 666 } 667 if chunkFilterer != nil && chunkFilterer.ShouldFilter(chunks[0][0].Chunk.Metric) { 668 removeSeries(fp, chunks) 669 continue outer 670 } 671 } 672 metrics.chunks.WithLabelValues(statusDiscarded).Add(float64(filteredChks)) 673 metrics.series.WithLabelValues(statusDiscarded).Add(float64(filteredSeries)) 674 return chks 675 } 676 677 func fetchLazyChunks(ctx context.Context, s config.SchemaConfig, chunks []*LazyChunk) error { 678 var ( 679 totalChunks int64 680 start = time.Now() 681 stats = stats.FromContext(ctx) 682 logger = util_log.WithContext(ctx, util_log.Logger) 683 ) 684 defer func() { 685 stats.AddChunksDownloadTime(time.Since(start)) 686 stats.AddChunksDownloaded(totalChunks) 687 }() 688 689 chksByFetcher := map[*fetcher.Fetcher][]*LazyChunk{} 690 for _, c := range chunks { 691 if c.Chunk.Data == nil { 692 chksByFetcher[c.Fetcher] = append(chksByFetcher[c.Fetcher], c) 693 totalChunks++ 694 } 695 } 696 if len(chksByFetcher) == 0 { 697 return nil 698 } 699 level.Debug(logger).Log("msg", "loading lazy chunks", "chunks", totalChunks) 700 701 errChan := make(chan error) 702 for f, chunks := range chksByFetcher { 703 go func(fetcher *fetcher.Fetcher, chunks []*LazyChunk) { 704 keys := make([]string, 0, len(chunks)) 705 chks := make([]chunk.Chunk, 0, len(chunks)) 706 index := make(map[string]*LazyChunk, len(chunks)) 707 708 // FetchChunks requires chunks to be ordered by external key. 709 sort.Slice(chunks, func(i, j int) bool { 710 return s.ExternalKey(chunks[i].Chunk.ChunkRef) < s.ExternalKey(chunks[j].Chunk.ChunkRef) 711 }) 712 for _, chk := range chunks { 713 key := s.ExternalKey(chk.Chunk.ChunkRef) 714 keys = append(keys, key) 715 chks = append(chks, chk.Chunk) 716 index[key] = chk 717 } 718 chks, err := fetcher.FetchChunks(ctx, chks, keys) 719 if err != nil { 720 level.Error(logger).Log("msg", "error fetching chunks", "err", err) 721 if isInvalidChunkError(err) { 722 level.Error(logger).Log("msg", "checksum of chunks does not match", "err", chunk.ErrInvalidChecksum) 723 errChan <- nil 724 return 725 } 726 errChan <- err 727 return 728 729 } 730 // assign fetched chunk by key as FetchChunks doesn't guarantee the order. 731 for _, chk := range chks { 732 index[s.ExternalKey(chk.ChunkRef)].Chunk = chk 733 } 734 735 errChan <- nil 736 }(f, chunks) 737 } 738 739 var lastErr error 740 for i := 0; i < len(chksByFetcher); i++ { 741 if err := <-errChan; err != nil { 742 lastErr = err 743 } 744 } 745 746 if lastErr != nil { 747 return lastErr 748 } 749 750 for _, c := range chunks { 751 if c.Chunk.Data != nil { 752 c.IsValid = true 753 } 754 } 755 return nil 756 } 757 758 func isInvalidChunkError(err error) bool { 759 err = errors.Cause(err) 760 if err, ok := err.(promql.ErrStorage); ok { 761 return err.Err == chunk.ErrInvalidChecksum || err.Err == chunkenc.ErrInvalidChecksum 762 } 763 return false 764 } 765 766 func loadFirstChunks(ctx context.Context, s config.SchemaConfig, chks map[model.Fingerprint][][]*LazyChunk) error { 767 var toLoad []*LazyChunk 768 for _, lchks := range chks { 769 for _, lchk := range lchks { 770 if len(lchk) == 0 { 771 continue 772 } 773 toLoad = append(toLoad, lchk[0]) 774 } 775 } 776 return fetchLazyChunks(ctx, s, toLoad) 777 } 778 779 func partitionBySeriesChunks(chunks []*LazyChunk) map[model.Fingerprint][][]*LazyChunk { 780 chunksByFp := map[model.Fingerprint][]*LazyChunk{} 781 for _, c := range chunks { 782 fp := c.Chunk.FingerprintModel() 783 chunksByFp[fp] = append(chunksByFp[fp], c) 784 } 785 result := make(map[model.Fingerprint][][]*LazyChunk, len(chunksByFp)) 786 787 for fp, chks := range chunksByFp { 788 result[fp] = partitionOverlappingChunks(chks) 789 } 790 791 return result 792 } 793 794 // partitionOverlappingChunks splits the list of chunks into different non-overlapping lists. 795 // todo this might reverse the order. 796 func partitionOverlappingChunks(chunks []*LazyChunk) [][]*LazyChunk { 797 sort.Slice(chunks, func(i, j int) bool { 798 return chunks[i].Chunk.From < chunks[j].Chunk.From 799 }) 800 801 css := [][]*LazyChunk{} 802 outer: 803 for _, c := range chunks { 804 for i, cs := range css { 805 // If the chunk doesn't overlap with the current list, then add it to it. 806 if cs[len(cs)-1].Chunk.Through.Before(c.Chunk.From) { 807 css[i] = append(css[i], c) 808 continue outer 809 } 810 } 811 // If the chunk overlaps with every existing list, then create a new list. 812 cs := make([]*LazyChunk, 0, len(chunks)/(len(css)+1)) 813 cs = append(cs, c) 814 css = append(css, cs) 815 } 816 817 return css 818 }