github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/tools/tsdb/tsdb-map/main.go (about)

     1  package main
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"flag"
     7  	"log"
     8  	"strconv"
     9  
    10  	"github.com/prometheus/common/model"
    11  	"go.etcd.io/bbolt"
    12  	"gopkg.in/yaml.v2"
    13  
    14  	"github.com/grafana/loki/pkg/storage/config"
    15  	"github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention"
    16  	"github.com/grafana/loki/pkg/storage/stores/shipper/index/compactor"
    17  	shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util"
    18  	"github.com/grafana/loki/pkg/storage/stores/tsdb"
    19  	"github.com/grafana/loki/pkg/storage/stores/tsdb/index"
    20  )
    21  
    22  var (
    23  	source = flag.String("source", "", "the source boltdb file")
    24  	dest   = flag.String("dest", "", "the dest tsdb dir")
    25  	// Hardcode a periodconfig for convenience as the boltdb iterator needs one
    26  	// NB: must match the index file you're reading from
    27  	periodConfig = func() config.PeriodConfig {
    28  		input := `
    29  from: "2022-01-01"
    30  index:
    31    period: 24h
    32    prefix: loki_index_
    33  object_store: gcs
    34  schema: v13
    35  store: boltdb-shipper
    36  `
    37  		var cfg config.PeriodConfig
    38  		if err := yaml.Unmarshal([]byte(input), &cfg); err != nil {
    39  			panic(err)
    40  		}
    41  		return cfg
    42  	}()
    43  )
    44  
    45  func extractChecksumFromChunkID(b []byte) uint32 {
    46  	i := bytes.LastIndexByte(b, ':')
    47  	x, err := strconv.ParseUint(string(b[i+1:]), 16, 32)
    48  	if err != nil {
    49  		panic(err)
    50  	}
    51  	return uint32(x)
    52  }
    53  
    54  func main() {
    55  	flag.Parse()
    56  
    57  	if source == nil || *source == "" {
    58  		panic("source is required")
    59  	}
    60  
    61  	if dest == nil || *dest == "" {
    62  		panic("dest is required")
    63  	}
    64  
    65  	db, err := shipper_util.SafeOpenBoltdbFile(*source)
    66  	if err != nil {
    67  		panic(err)
    68  	}
    69  
    70  	builder := tsdb.NewBuilder()
    71  
    72  	log.Println("Loading index into memory")
    73  
    74  	// loads everything into memory.
    75  	if err := db.View(func(t *bbolt.Tx) error {
    76  		return compactor.ForEachChunk(context.Background(), t.Bucket([]byte("index")), periodConfig, func(entry retention.ChunkEntry) (bool, error) {
    77  			builder.AddSeries(entry.Labels, model.Fingerprint(entry.Labels.Hash()), []index.ChunkMeta{{
    78  				Checksum: extractChecksumFromChunkID(entry.ChunkID),
    79  				MinTime:  int64(entry.From),
    80  				MaxTime:  int64(entry.Through),
    81  				KB:       ((3 << 20) / 4) / 1024, // guess: 0.75mb, 1/2 of the max size, rounded to KB
    82  				Entries:  10000,                  // guess: 10k entries
    83  			}})
    84  			return false, nil
    85  		})
    86  	}); err != nil {
    87  		panic(err)
    88  	}
    89  
    90  	log.Println("writing index")
    91  	if _, err := builder.Build(context.Background(), *dest, func(from, through model.Time, checksum uint32) tsdb.Identifier {
    92  		panic("todo")
    93  	}); err != nil {
    94  		panic(err)
    95  	}
    96  }