github.com/grafana/pyroscope@v1.18.0/pkg/ingester/instance.go (about)

     1  package ingester
     2  
     3  import (
     4  	"context"
     5  	"path"
     6  	"sync"
     7  	"time"
     8  
     9  	"github.com/go-kit/log"
    10  	"github.com/go-kit/log/level"
    11  	"github.com/prometheus/client_golang/prometheus"
    12  
    13  	phlareobj "github.com/grafana/pyroscope/pkg/objstore"
    14  	"github.com/grafana/pyroscope/pkg/phlaredb"
    15  	"github.com/grafana/pyroscope/pkg/phlaredb/block"
    16  	"github.com/grafana/pyroscope/pkg/phlaredb/shipper"
    17  	phlarecontext "github.com/grafana/pyroscope/pkg/pyroscope/context"
    18  )
    19  
    20  type instance struct {
    21  	*phlaredb.PhlareDB
    22  	shipper     *shipper.Shipper
    23  	shipperLock sync.Mutex
    24  	logger      log.Logger
    25  	reg         prometheus.Registerer
    26  
    27  	cancel   context.CancelFunc
    28  	wg       sync.WaitGroup
    29  	tenantID string
    30  }
    31  
    32  func newInstance(phlarectx context.Context, cfg phlaredb.Config, tenantID string, localBucket, storageBucket phlareobj.Bucket, limiter Limiter) (*instance, error) {
    33  	cfg.DataPath = path.Join(cfg.DataPath, tenantID)
    34  
    35  	// TODO(kolesnikovae): Get rid of phlarectx and pass logger and registry directly.
    36  	phlarectx = phlarecontext.WrapTenant(phlarectx, tenantID)
    37  	reg := prometheus.WrapRegistererWith(prometheus.Labels{"component": "ingester"}, phlarecontext.Registry(phlarectx))
    38  	phlarectx = phlarecontext.WithRegistry(phlarectx, reg)
    39  
    40  	db, err := phlaredb.New(phlarectx, cfg, limiter, phlareobj.NewPrefixedBucket(localBucket, tenantID))
    41  	if err != nil {
    42  		return nil, err
    43  	}
    44  	ctx, cancel := context.WithCancel(phlarectx)
    45  	inst := &instance{
    46  		PhlareDB: db,
    47  		logger:   phlarecontext.Logger(phlarectx),
    48  		reg:      reg,
    49  		cancel:   cancel,
    50  		tenantID: tenantID,
    51  	}
    52  	// Todo we should not ship when using filesystem storage.
    53  	if storageBucket != nil {
    54  		inst.shipper = shipper.New(
    55  			inst.logger,
    56  			inst.reg,
    57  			db,
    58  			phlareobj.NewTenantBucketClient(tenantID, storageBucket, nil),
    59  			block.IngesterSource,
    60  			false,
    61  			false,
    62  		)
    63  	}
    64  	go inst.loop(ctx)
    65  	return inst, nil
    66  }
    67  
    68  func (i *instance) loop(ctx context.Context) {
    69  	i.wg.Add(1)
    70  	defer func() {
    71  		i.runShipper(context.Background()) // Run shipper one last time.
    72  		i.wg.Done()
    73  	}()
    74  	// run shipper periodically and at start-up
    75  	shipperTicker := time.NewTicker(5 * time.Minute)
    76  	defer shipperTicker.Stop()
    77  	go func() {
    78  		i.runShipper(ctx)
    79  	}()
    80  
    81  	for {
    82  		select {
    83  		case <-ctx.Done():
    84  			return
    85  		case <-shipperTicker.C: // run shipper loop
    86  			i.runShipper(ctx)
    87  		}
    88  	}
    89  }
    90  
    91  func (i *instance) runShipper(ctx context.Context) {
    92  	i.shipperLock.Lock()
    93  	defer i.shipperLock.Unlock()
    94  	if i.shipper == nil {
    95  		return
    96  	}
    97  	uploaded, err := i.shipper.Sync(ctx)
    98  	if err != nil {
    99  		level.Error(i.logger).Log("msg", "shipper run failed", "err", err)
   100  	} else {
   101  		level.Info(i.logger).Log("msg", "shipper finished", "uploaded_blocks", uploaded)
   102  	}
   103  }
   104  
   105  func (i *instance) Stop() error {
   106  	err := i.Close()
   107  	i.cancel()
   108  	i.wg.Wait()
   109  	return err
   110  }