github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/stores/tsdb/compact.go (about) 1 package tsdb 2 3 import ( 4 "context" 5 "fmt" 6 "time" 7 8 "github.com/prometheus/common/model" 9 "github.com/prometheus/prometheus/model/labels" 10 11 "github.com/grafana/loki/pkg/storage/stores/tsdb/index" 12 ) 13 14 type Compactor struct { 15 // metrics 16 // file names (tenant, bounds, checksum?) 17 tenant string 18 parentDir string 19 } 20 21 func NewCompactor(tenant, parentDir string) *Compactor { 22 return &Compactor{ 23 tenant: tenant, 24 parentDir: parentDir, 25 } 26 } 27 28 func (c *Compactor) Compact(ctx context.Context, indices ...*TSDBIndex) (res Identifier, err error) { 29 // No need to compact a single index file 30 if len(indices) == 1 { 31 return newPrefixedIdentifier( 32 indices[0].Identifier(c.tenant), 33 c.parentDir, 34 c.parentDir, 35 ), 36 nil 37 } 38 39 ifcs := make([]Index, 0, len(indices)) 40 for _, idx := range indices { 41 ifcs = append(ifcs, idx) 42 } 43 44 b := NewBuilder() 45 multi, err := NewMultiIndex(ifcs...) 46 if err != nil { 47 return res, err 48 } 49 50 // TODO(owen-d): introduce parallelism 51 // Until then, 52 // Instead of using the MultiIndex.forIndices helper, we loop over each sub-index manually. 53 // The index builder is single threaded, so we avoid races. 54 // Additionally, this increases the likelihood we add chunks in order 55 // by processing the indices in ascending order. 56 for _, idx := range multi.indices { 57 casted, ok := idx.(*TSDBIndex) 58 if !ok { 59 return nil, fmt.Errorf("expected tsdb index to compact, found :%T", idx) 60 } 61 if err := casted.forSeries( 62 ctx, 63 nil, 64 func(ls labels.Labels, _ model.Fingerprint, chks []index.ChunkMeta) { 65 // AddSeries copies chks into it's own slice 66 b.AddSeries(ls.Copy(), model.Fingerprint(ls.Hash()), chks) 67 }, 68 labels.MustNewMatcher(labels.MatchEqual, "", ""), 69 ); err != nil { 70 return res, err 71 } 72 } 73 74 return b.Build( 75 ctx, 76 c.parentDir, 77 func(from, through model.Time, checksum uint32) Identifier { 78 id := SingleTenantTSDBIdentifier{ 79 TS: time.Now(), 80 From: from, 81 Through: through, 82 Checksum: checksum, 83 } 84 return newPrefixedIdentifier(id, c.parentDir, c.parentDir) 85 }, 86 ) 87 }