github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/stores/tsdb/chunkwriter.go (about)

     1  package tsdb
     2  
     3  import (
     4  	"context"
     5  	"math"
     6  
     7  	"github.com/go-kit/log/level"
     8  	"github.com/pkg/errors"
     9  	"github.com/prometheus/common/model"
    10  	"github.com/prometheus/prometheus/model/labels"
    11  
    12  	"github.com/grafana/loki/pkg/storage/chunk"
    13  	"github.com/grafana/loki/pkg/storage/chunk/fetcher"
    14  	"github.com/grafana/loki/pkg/storage/config"
    15  	"github.com/grafana/loki/pkg/storage/stores/tsdb/index"
    16  	"github.com/grafana/loki/pkg/util/spanlogger"
    17  )
    18  
    19  type IndexWriter interface {
    20  	Append(userID string, ls labels.Labels, chks index.ChunkMetas) error
    21  }
    22  
    23  type ChunkWriter struct {
    24  	schemaCfg   config.SchemaConfig
    25  	fetcher     *fetcher.Fetcher
    26  	indexWriter IndexWriter
    27  }
    28  
    29  func NewChunkWriter(
    30  	fetcher *fetcher.Fetcher,
    31  	pd config.PeriodConfig,
    32  	indexWriter IndexWriter,
    33  ) *ChunkWriter {
    34  	return &ChunkWriter{
    35  		schemaCfg: config.SchemaConfig{
    36  			Configs: []config.PeriodConfig{pd},
    37  		},
    38  		fetcher:     fetcher,
    39  		indexWriter: indexWriter,
    40  	}
    41  }
    42  
    43  func (w *ChunkWriter) Put(ctx context.Context, chunks []chunk.Chunk) error {
    44  	for _, chunk := range chunks {
    45  		if err := w.PutOne(ctx, chunk.From, chunk.Through, chunk); err != nil {
    46  			return err
    47  		}
    48  	}
    49  	return nil
    50  }
    51  
    52  func (w *ChunkWriter) PutOne(ctx context.Context, from, through model.Time, chk chunk.Chunk) error {
    53  	log, ctx := spanlogger.New(ctx, "TSDBStore.PutOne")
    54  	defer log.Finish()
    55  
    56  	// with local TSDB indices, we _always_ write the index entry
    57  	// to avoid data loss if we lose an ingester's disk
    58  	// but we can skip writing the chunk if another replica
    59  	// has already written it to storage.
    60  	chunks := []chunk.Chunk{chk}
    61  
    62  	c := w.fetcher.Client()
    63  	if err := c.PutChunks(ctx, chunks); err != nil {
    64  		return errors.Wrap(err, "writing chunk")
    65  	}
    66  
    67  	// Always write the index to benefit durability via replication factor.
    68  	approxKB := math.Round(float64(chk.Data.UncompressedSize()) / float64(1<<10))
    69  	metas := index.ChunkMetas{
    70  		{
    71  			Checksum: chk.ChunkRef.Checksum,
    72  			MinTime:  int64(chk.ChunkRef.From),
    73  			MaxTime:  int64(chk.ChunkRef.Through),
    74  			KB:       uint32(approxKB),
    75  			Entries:  uint32(chk.Data.Entries()),
    76  		},
    77  	}
    78  	if err := w.indexWriter.Append(chk.UserID, chk.Metric, metas); err != nil {
    79  		return errors.Wrap(err, "writing index entry")
    80  	}
    81  
    82  	if cacheErr := w.fetcher.WriteBackCache(ctx, chunks); cacheErr != nil {
    83  		level.Warn(log).Log("msg", "could not store chunks in chunk cache", "err", cacheErr)
    84  	}
    85  
    86  	return nil
    87  }