github.com/grafana/pyroscope@v1.18.0/cmd/profilecli/compact.go (about)

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"path/filepath"
     8  	"time"
     9  
    10  	"github.com/briandowns/spinner"
    11  	"github.com/dustin/go-humanize"
    12  	"github.com/go-kit/log"
    13  	"github.com/olekukonko/tablewriter"
    14  	"github.com/pkg/errors"
    15  	"golang.org/x/sync/errgroup"
    16  
    17  	"github.com/grafana/pyroscope/pkg/objstore/client"
    18  	"github.com/grafana/pyroscope/pkg/objstore/providers/filesystem"
    19  	"github.com/grafana/pyroscope/pkg/phlaredb"
    20  	"github.com/grafana/pyroscope/pkg/phlaredb/block"
    21  	phlarecontext "github.com/grafana/pyroscope/pkg/pyroscope/context"
    22  )
    23  
    24  func blocksCompact(ctx context.Context, src, dst string, shards int) error {
    25  	var inputBlocks []*block.Meta
    26  	_, ok := block.IsBlockDir(src)
    27  	if ok {
    28  		meta, err := block.ReadMetaFromDir(src)
    29  		if err != nil {
    30  			return err
    31  		}
    32  		src = filepath.Clean(filepath.Join(src, "/../"))
    33  		inputBlocks = append(inputBlocks, meta)
    34  
    35  	} else {
    36  		blocks, err := block.ListBlocks(src, time.Time{})
    37  		if err != nil {
    38  			return err
    39  		}
    40  		if len(blocks) == 0 {
    41  			return errors.New("no input blocks found, please provide either a single block directory or a directory containing blocks")
    42  		}
    43  		for _, b := range blocks {
    44  			inputBlocks = append(inputBlocks, b.Clone())
    45  		}
    46  	}
    47  	return compact(ctx, src, dst, inputBlocks, shards)
    48  }
    49  
    50  func compact(ctx context.Context, src, dst string, metas []*block.Meta, shards int) error {
    51  	// create the destination directory if it doesn't exist
    52  	if _, err := os.Stat(dst); errors.Is(err, os.ErrNotExist) {
    53  		if err := os.MkdirAll(dst, 0o755); err != nil {
    54  			return errors.Wrap(err, "create dir")
    55  		}
    56  	}
    57  
    58  	ctx = phlarecontext.WithLogger(ctx, log.NewNopLogger())
    59  	blocks := make([]phlaredb.BlockReader, 0, len(metas))
    60  	in := make([]block.Meta, 0, len(metas))
    61  
    62  	bkt, err := client.NewBucket(ctx, client.Config{
    63  		StorageBackendConfig: client.StorageBackendConfig{
    64  			Backend: client.Filesystem,
    65  			Filesystem: filesystem.Config{
    66  				Directory: src,
    67  			},
    68  		},
    69  		Prefix: "",
    70  	}, "profilecli")
    71  	if err != nil {
    72  		return err
    73  	}
    74  
    75  	for _, m := range metas {
    76  		in = append(in, *m)
    77  		blocks = append(blocks, phlaredb.NewSingleBlockQuerierFromMeta(ctx, bkt, m))
    78  	}
    79  	fmt.Fprintln(output(ctx), "Found Input blocks:")
    80  	printMeta(ctx, in)
    81  	s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriter(output(ctx)))
    82  	s.Suffix = " Loading data..."
    83  	s.Start()
    84  	g, groupCtx := errgroup.WithContext(ctx)
    85  
    86  	for _, b := range blocks {
    87  		b := b
    88  		g.Go(func() error {
    89  			return b.Open(groupCtx)
    90  		})
    91  	}
    92  	if err := g.Wait(); err != nil {
    93  		s.Stop()
    94  		return err
    95  	}
    96  
    97  	s.Suffix = " Compacting data..."
    98  	s.Restart()
    99  
   100  	out, err := phlaredb.CompactWithSplitting(ctx, phlaredb.CompactWithSplittingOpts{
   101  		Src:                blocks,
   102  		Dst:                dst,
   103  		SplitCount:         uint64(shards),
   104  		StageSize:          0,
   105  		SplitBy:            phlaredb.SplitByFingerprint,
   106  		DownsamplerEnabled: true,
   107  		Logger:             logger,
   108  	})
   109  	if err != nil {
   110  		s.Stop()
   111  		return err
   112  	}
   113  
   114  	s.Stop()
   115  	fmt.Fprintln(output(ctx), "Output blocks:")
   116  	printMeta(ctx, out)
   117  	return nil
   118  }
   119  
   120  func printMeta(ctx context.Context, metas []block.Meta) {
   121  	table := tablewriter.NewWriter(output(ctx))
   122  	table.SetHeader([]string{"Block ID", "MinTime", "MaxTime", "Duration", "Index", "Profiles", "Symbols", "Labels"})
   123  	for _, blockInfo := range metas {
   124  		table.Append([]string{
   125  			blockInfo.ULID.String(),
   126  			blockInfo.MinTime.Time().Format(time.RFC3339),
   127  			blockInfo.MaxTime.Time().Format(time.RFC3339),
   128  			blockInfo.MaxTime.Time().Sub(blockInfo.MinTime.Time()).String(),
   129  			indexInfo(blockInfo),
   130  			fileInfo(blockInfo.FileByRelPath("profiles.parquet")),
   131  			symbolSize(blockInfo),
   132  			labelsString(blockInfo.Labels),
   133  		})
   134  	}
   135  	table.Render()
   136  }
   137  
   138  func indexInfo(meta block.Meta) string {
   139  	if f := meta.FileByRelPath("index.tsdb"); f != nil {
   140  		return fmt.Sprintf("%d series (%s)", f.TSDB.NumSeries, humanize.Bytes(f.SizeBytes))
   141  	}
   142  	return ""
   143  }
   144  
   145  func symbolSize(meta block.Meta) string {
   146  	size := uint64(0)
   147  	if f := meta.FileByRelPath("symbols/index.symdb"); f != nil {
   148  		size += f.SizeBytes
   149  	}
   150  	if f := meta.FileByRelPath("symbols/stacktraces.symdb"); f != nil {
   151  		size += f.SizeBytes
   152  	}
   153  	if f := meta.FileByRelPath("symbols/locations.parquet"); f != nil {
   154  		size += f.SizeBytes
   155  	}
   156  	if f := meta.FileByRelPath("symbols/functions.parquet"); f != nil {
   157  		size += f.SizeBytes
   158  	}
   159  	if f := meta.FileByRelPath("symbols/mapping.parquet"); f != nil {
   160  		size += f.SizeBytes
   161  	}
   162  	if f := meta.FileByRelPath("symbols/strings.parquet"); f != nil {
   163  		size += f.SizeBytes
   164  	}
   165  
   166  	return humanize.Bytes(size)
   167  }
   168  
   169  func labelsString(m map[string]string) string {
   170  	var s string
   171  	for k, v := range m {
   172  		s += fmt.Sprintf("%s=%s,", k, v)
   173  	}
   174  	return s
   175  }