github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/querier/queryrange/shard_resolver.go (about)

     1  package queryrange
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	math "math"
     7  	strings "strings"
     8  	"time"
     9  
    10  	"github.com/dustin/go-humanize"
    11  	"github.com/go-kit/log"
    12  	"github.com/go-kit/log/level"
    13  	"github.com/grafana/dskit/concurrency"
    14  	"github.com/prometheus/common/model"
    15  
    16  	"github.com/grafana/loki/pkg/logproto"
    17  	"github.com/grafana/loki/pkg/logql"
    18  	"github.com/grafana/loki/pkg/logql/syntax"
    19  	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
    20  	"github.com/grafana/loki/pkg/storage/config"
    21  	"github.com/grafana/loki/pkg/storage/stores/index/stats"
    22  	"github.com/grafana/loki/pkg/util/spanlogger"
    23  )
    24  
    25  func shardResolverForConf(
    26  	ctx context.Context,
    27  	conf config.PeriodConfig,
    28  	defaultLookback time.Duration,
    29  	logger log.Logger,
    30  	maxParallelism int,
    31  	r queryrangebase.Request,
    32  	handler queryrangebase.Handler,
    33  ) (logql.ShardResolver, bool) {
    34  	if conf.IndexType == config.TSDBType {
    35  		return &dynamicShardResolver{
    36  			ctx:             ctx,
    37  			logger:          logger,
    38  			handler:         handler,
    39  			from:            model.Time(r.GetStart()),
    40  			through:         model.Time(r.GetEnd()),
    41  			maxParallelism:  maxParallelism,
    42  			defaultLookback: defaultLookback,
    43  		}, true
    44  	}
    45  	if conf.RowShards < 2 {
    46  		return nil, false
    47  	}
    48  	return logql.ConstantShards(conf.RowShards), true
    49  }
    50  
    51  type dynamicShardResolver struct {
    52  	ctx     context.Context
    53  	handler queryrangebase.Handler
    54  	logger  log.Logger
    55  
    56  	from, through   model.Time
    57  	maxParallelism  int
    58  	defaultLookback time.Duration
    59  }
    60  
    61  func (r *dynamicShardResolver) Shards(e syntax.Expr) (int, error) {
    62  	sp, ctx := spanlogger.NewWithLogger(r.ctx, r.logger, "dynamicShardResolver.Shards")
    63  	defer sp.Finish()
    64  	// We try to shard subtrees in the AST independently if possible, although
    65  	// nested binary expressions can make this difficult. In this case,
    66  	// we query the index stats for all matcher groups then sum the results.
    67  	grps := syntax.MatcherGroups(e)
    68  
    69  	// If there are zero matchers groups, we'll inject one to query everything
    70  	if len(grps) == 0 {
    71  		grps = append(grps, syntax.MatcherRange{})
    72  	}
    73  
    74  	results := make([]*stats.Stats, 0, len(grps))
    75  
    76  	start := time.Now()
    77  	if err := concurrency.ForEachJob(ctx, len(grps), r.maxParallelism, func(ctx context.Context, i int) error {
    78  		matchers := syntax.MatchersString(grps[i].Matchers)
    79  		diff := grps[i].Interval + grps[i].Offset
    80  		adjustedFrom := r.from.Add(-diff)
    81  		if grps[i].Interval == 0 {
    82  			adjustedFrom = adjustedFrom.Add(-r.defaultLookback)
    83  		}
    84  
    85  		adjustedThrough := r.through.Add(-grps[i].Offset)
    86  
    87  		start := time.Now()
    88  		resp, err := r.handler.Do(r.ctx, &logproto.IndexStatsRequest{
    89  			From:     adjustedFrom,
    90  			Through:  adjustedThrough,
    91  			Matchers: matchers,
    92  		})
    93  		if err != nil {
    94  			return err
    95  		}
    96  
    97  		casted, ok := resp.(*IndexStatsResponse)
    98  		if !ok {
    99  			return fmt.Errorf("expected *IndexStatsResponse while querying index, got %T", resp)
   100  		}
   101  
   102  		results = append(results, casted.Response)
   103  		level.Debug(sp).Log(
   104  			"msg", "queried index",
   105  			"type", "single",
   106  			"matchers", matchers,
   107  			"bytes", strings.Replace(humanize.Bytes(casted.Response.Bytes), " ", "", 1),
   108  			"chunks", casted.Response.Chunks,
   109  			"streams", casted.Response.Streams,
   110  			"entries", casted.Response.Entries,
   111  			"duration", time.Since(start),
   112  			"from", adjustedFrom.Time(),
   113  			"through", adjustedThrough.Time(),
   114  			"length", adjustedThrough.Sub(adjustedFrom),
   115  		)
   116  		return nil
   117  	}); err != nil {
   118  		return 0, err
   119  	}
   120  
   121  	combined := stats.MergeStats(results...)
   122  	factor := guessShardFactor(combined)
   123  	var bytesPerShard = combined.Bytes
   124  	if factor > 0 {
   125  		bytesPerShard = combined.Bytes / uint64(factor)
   126  	}
   127  	level.Debug(sp).Log(
   128  		"msg", "queried index",
   129  		"type", "combined",
   130  		"len", len(results),
   131  		"bytes", strings.Replace(humanize.Bytes(combined.Bytes), " ", "", 1),
   132  		"chunks", combined.Chunks,
   133  		"streams", combined.Streams,
   134  		"entries", combined.Entries,
   135  		"max_parallelism", r.maxParallelism,
   136  		"duration", time.Since(start),
   137  		"factor", factor,
   138  		"bytes_per_shard", strings.Replace(humanize.Bytes(bytesPerShard), " ", "", 1),
   139  	)
   140  	return factor, nil
   141  }
   142  
   143  const (
   144  	// Just some observed values to get us started on better query planning.
   145  	p90BytesPerSecond = 300 << 20 // 300MB/s/core
   146  )
   147  
   148  func guessShardFactor(stats stats.Stats) int {
   149  	expectedSeconds := float64(stats.Bytes) / float64(p90BytesPerSecond)
   150  	power := math.Ceil(math.Log2(expectedSeconds)) // round up to nearest power of 2
   151  
   152  	// Parallelize down to 1s queries
   153  	// Since x^0 == 1 and we only support factors of 2
   154  	// reset this edge case manually
   155  	factor := int(math.Pow(2, power))
   156  	if factor == 1 {
   157  		factor = 0
   158  	}
   159  	return factor
   160  }