github.com/sequix/cortex@v1.1.6/pkg/chunk/util/parallel_chunk_fetch.go (about)

     1  package util
     2  
     3  import (
     4  	"context"
     5  
     6  	ot "github.com/opentracing/opentracing-go"
     7  	otlog "github.com/opentracing/opentracing-go/log"
     8  
     9  	"github.com/sequix/cortex/pkg/chunk"
    10  )
    11  
    12  const maxParallel = 1000
    13  
    14  // GetParallelChunks fetches chunks in parallel (up to maxParallel).
    15  func GetParallelChunks(ctx context.Context, chunks []chunk.Chunk, f func(context.Context, *chunk.DecodeContext, chunk.Chunk) (chunk.Chunk, error)) ([]chunk.Chunk, error) {
    16  	sp, ctx := ot.StartSpanFromContext(ctx, "GetParallelChunks")
    17  	defer sp.Finish()
    18  	sp.LogFields(otlog.Int("chunks requested", len(chunks)))
    19  
    20  	queuedChunks := make(chan chunk.Chunk)
    21  
    22  	go func() {
    23  		for _, c := range chunks {
    24  			queuedChunks <- c
    25  		}
    26  		close(queuedChunks)
    27  	}()
    28  
    29  	processedChunks := make(chan chunk.Chunk)
    30  	errors := make(chan error)
    31  
    32  	for i := 0; i < min(maxParallel, len(chunks)); i++ {
    33  		go func() {
    34  			decodeContext := chunk.NewDecodeContext()
    35  			for c := range queuedChunks {
    36  				c, err := f(ctx, decodeContext, c)
    37  				if err != nil {
    38  					errors <- err
    39  				} else {
    40  					processedChunks <- c
    41  				}
    42  			}
    43  		}()
    44  	}
    45  
    46  	var result = make([]chunk.Chunk, 0, len(chunks))
    47  	var lastErr error
    48  	for i := 0; i < len(chunks); i++ {
    49  		select {
    50  		case chunk := <-processedChunks:
    51  			result = append(result, chunk)
    52  		case err := <-errors:
    53  			lastErr = err
    54  		}
    55  	}
    56  
    57  	sp.LogFields(otlog.Int("chunks fetched", len(result)))
    58  	if lastErr != nil {
    59  		sp.LogFields(otlog.Error(lastErr))
    60  	}
    61  
    62  	// Return any chunks we did receive: a partial result may be useful
    63  	return result, lastErr
    64  }
    65  
    66  func min(a, b int) int {
    67  	if a < b {
    68  		return a
    69  	}
    70  	return b
    71  }