github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/hack/main.go (about)

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"log"
     7  	"math/rand"
     8  	"os"
     9  	"sync"
    10  	"time"
    11  
    12  	"github.com/prometheus/client_golang/prometheus"
    13  	"github.com/prometheus/common/model"
    14  	"github.com/prometheus/prometheus/model/labels"
    15  	"github.com/weaveworks/common/user"
    16  
    17  	"github.com/grafana/loki/pkg/chunkenc"
    18  	"github.com/grafana/loki/pkg/ingester/client"
    19  	"github.com/grafana/loki/pkg/logproto"
    20  	"github.com/grafana/loki/pkg/logql/syntax"
    21  	"github.com/grafana/loki/pkg/storage"
    22  	"github.com/grafana/loki/pkg/storage/chunk"
    23  	"github.com/grafana/loki/pkg/storage/chunk/client/local"
    24  	"github.com/grafana/loki/pkg/storage/config"
    25  	util_log "github.com/grafana/loki/pkg/util/log"
    26  	"github.com/grafana/loki/pkg/validation"
    27  )
    28  
    29  var (
    30  	start     = model.Time(1523750400000)
    31  	ctx       = user.InjectOrgID(context.Background(), "fake")
    32  	maxChunks = 1200 // 1200 chunks is 2gib ish of data enough to run benchmark
    33  )
    34  
    35  // fill up the local filesystem store with 1gib of data to run benchmark
    36  func main() {
    37  	cm := storage.NewClientMetrics()
    38  	defer cm.Unregister()
    39  	if _, err := os.Stat("/tmp/benchmark/chunks"); os.IsNotExist(err) {
    40  		if err := fillStore(cm); err != nil {
    41  			log.Fatal("error filling up storage:", err)
    42  		}
    43  	}
    44  }
    45  
    46  func getStore(cm storage.ClientMetrics) (storage.Store, error) {
    47  	storeConfig := storage.Config{
    48  		BoltDBConfig: local.BoltDBConfig{Directory: "/tmp/benchmark/index"},
    49  		FSConfig:     local.FSConfig{Directory: "/tmp/benchmark/chunks"},
    50  	}
    51  
    52  	schemaCfg := config.SchemaConfig{
    53  		Configs: []config.PeriodConfig{
    54  			{
    55  				From:       config.DayTime{Time: start},
    56  				IndexType:  "boltdb",
    57  				ObjectType: "filesystem",
    58  				Schema:     "v9",
    59  				IndexTables: config.PeriodicTableConfig{
    60  					Prefix: "index_",
    61  					Period: time.Hour * 168,
    62  				},
    63  			},
    64  		},
    65  	}
    66  
    67  	return storage.NewStore(storeConfig, config.ChunkStoreConfig{}, schemaCfg, &validation.Overrides{}, cm, prometheus.DefaultRegisterer, util_log.Logger)
    68  }
    69  
    70  func fillStore(cm storage.ClientMetrics) error {
    71  	store, err := getStore(cm)
    72  	if err != nil {
    73  		return err
    74  	}
    75  	defer store.Stop()
    76  
    77  	var wgPush sync.WaitGroup
    78  	var flushCount int
    79  	// insert 5 streams with a random logs every nanoseconds
    80  	// the string is randomize so chunks are big ~2mb
    81  	// take ~1min to build 1gib of data
    82  	for i := 0; i < 5; i++ {
    83  		wgPush.Add(1)
    84  		go func(j int) {
    85  			defer wgPush.Done()
    86  			lbs, err := syntax.ParseLabels(fmt.Sprintf("{foo=\"bar\",level=\"%d\"}", j))
    87  			if err != nil {
    88  				panic(err)
    89  			}
    90  			labelsBuilder := labels.NewBuilder(lbs)
    91  			labelsBuilder.Set(labels.MetricName, "logs")
    92  			metric := labelsBuilder.Labels()
    93  			fp := client.Fingerprint(lbs)
    94  			chunkEnc := chunkenc.NewMemChunk(chunkenc.EncLZ4_4M, chunkenc.UnorderedHeadBlockFmt, 262144, 1572864)
    95  			for ts := start.UnixNano(); ts < start.UnixNano()+time.Hour.Nanoseconds(); ts = ts + time.Millisecond.Nanoseconds() {
    96  				entry := &logproto.Entry{
    97  					Timestamp: time.Unix(0, ts),
    98  					Line:      randString(250),
    99  				}
   100  				if chunkEnc.SpaceFor(entry) {
   101  					_ = chunkEnc.Append(entry)
   102  				} else {
   103  					from, to := chunkEnc.Bounds()
   104  					c := chunk.NewChunk("fake", fp, metric, chunkenc.NewFacade(chunkEnc, 0, 0), model.TimeFromUnixNano(from.UnixNano()), model.TimeFromUnixNano(to.UnixNano()))
   105  					if err := c.Encode(); err != nil {
   106  						panic(err)
   107  					}
   108  					err := store.Put(ctx, []chunk.Chunk{c})
   109  					if err != nil {
   110  						panic(err)
   111  					}
   112  					flushCount++
   113  					log.Println("flushed ", flushCount, from.UnixNano(), to.UnixNano(), metric)
   114  					if flushCount >= maxChunks {
   115  						return
   116  					}
   117  					chunkEnc = chunkenc.NewMemChunk(chunkenc.EncLZ4_64k, chunkenc.UnorderedHeadBlockFmt, 262144, 1572864)
   118  				}
   119  			}
   120  		}(i)
   121  
   122  	}
   123  	wgPush.Wait()
   124  	return nil
   125  }
   126  
   127  const charset = "abcdefghijklmnopqrstuvwxyz" +
   128  	"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
   129  
   130  func randStringWithCharset(length int, charset string) string {
   131  	b := make([]byte, length)
   132  	for i := range b {
   133  		b[i] = charset[rand.Intn(len(charset)-1)]
   134  	}
   135  	return string(b)
   136  }
   137  
   138  func randString(length int) string {
   139  	return randStringWithCharset(length, charset)
   140  }