github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/stores/tsdb/compact.go (about)

     1  package tsdb
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/prometheus/common/model"
     9  	"github.com/prometheus/prometheus/model/labels"
    10  
    11  	"github.com/grafana/loki/pkg/storage/stores/tsdb/index"
    12  )
    13  
    14  type Compactor struct {
    15  	// metrics
    16  	// file names (tenant, bounds, checksum?)
    17  	tenant    string
    18  	parentDir string
    19  }
    20  
    21  func NewCompactor(tenant, parentDir string) *Compactor {
    22  	return &Compactor{
    23  		tenant:    tenant,
    24  		parentDir: parentDir,
    25  	}
    26  }
    27  
    28  func (c *Compactor) Compact(ctx context.Context, indices ...*TSDBIndex) (res Identifier, err error) {
    29  	// No need to compact a single index file
    30  	if len(indices) == 1 {
    31  		return newPrefixedIdentifier(
    32  				indices[0].Identifier(c.tenant),
    33  				c.parentDir,
    34  				c.parentDir,
    35  			),
    36  			nil
    37  	}
    38  
    39  	ifcs := make([]Index, 0, len(indices))
    40  	for _, idx := range indices {
    41  		ifcs = append(ifcs, idx)
    42  	}
    43  
    44  	b := NewBuilder()
    45  	multi, err := NewMultiIndex(ifcs...)
    46  	if err != nil {
    47  		return res, err
    48  	}
    49  
    50  	// TODO(owen-d): introduce parallelism
    51  	// Until then,
    52  	// Instead of using the MultiIndex.forIndices helper, we loop over each sub-index manually.
    53  	// The index builder is single threaded, so we avoid races.
    54  	// Additionally, this increases the likelihood we add chunks in order
    55  	// by processing the indices in ascending order.
    56  	for _, idx := range multi.indices {
    57  		casted, ok := idx.(*TSDBIndex)
    58  		if !ok {
    59  			return nil, fmt.Errorf("expected tsdb index to compact, found :%T", idx)
    60  		}
    61  		if err := casted.forSeries(
    62  			ctx,
    63  			nil,
    64  			func(ls labels.Labels, _ model.Fingerprint, chks []index.ChunkMeta) {
    65  				// AddSeries copies chks into it's own slice
    66  				b.AddSeries(ls.Copy(), model.Fingerprint(ls.Hash()), chks)
    67  			},
    68  			labels.MustNewMatcher(labels.MatchEqual, "", ""),
    69  		); err != nil {
    70  			return res, err
    71  		}
    72  	}
    73  
    74  	return b.Build(
    75  		ctx,
    76  		c.parentDir,
    77  		func(from, through model.Time, checksum uint32) Identifier {
    78  			id := SingleTenantTSDBIdentifier{
    79  				TS:       time.Now(),
    80  				From:     from,
    81  				Through:  through,
    82  				Checksum: checksum,
    83  			}
    84  			return newPrefixedIdentifier(id, c.parentDir, c.parentDir)
    85  		},
    86  	)
    87  }