github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/storage/tsdb/index_cache.go (about)

     1  package tsdb
     2  
     3  import (
     4  	"flag"
     5  	"fmt"
     6  	"strings"
     7  
     8  	"github.com/alecthomas/units"
     9  	"github.com/go-kit/log"
    10  	"github.com/pkg/errors"
    11  	"github.com/prometheus/client_golang/prometheus"
    12  	"github.com/thanos-io/thanos/pkg/cacheutil"
    13  	"github.com/thanos-io/thanos/pkg/model"
    14  	storecache "github.com/thanos-io/thanos/pkg/store/cache"
    15  
    16  	"github.com/cortexproject/cortex/pkg/util"
    17  )
    18  
    19  const (
    20  	// IndexCacheBackendInMemory is the value for the in-memory index cache backend.
    21  	IndexCacheBackendInMemory = "inmemory"
    22  
    23  	// IndexCacheBackendMemcached is the value for the memcached index cache backend.
    24  	IndexCacheBackendMemcached = "memcached"
    25  
    26  	// IndexCacheBackendDefault is the value for the default index cache backend.
    27  	IndexCacheBackendDefault = IndexCacheBackendInMemory
    28  
    29  	defaultMaxItemSize = model.Bytes(128 * units.MiB)
    30  )
    31  
    32  var (
    33  	supportedIndexCacheBackends = []string{IndexCacheBackendInMemory, IndexCacheBackendMemcached}
    34  
    35  	errUnsupportedIndexCacheBackend = errors.New("unsupported index cache backend")
    36  	errNoIndexCacheAddresses        = errors.New("no index cache backend addresses")
    37  )
    38  
    39  type IndexCacheConfig struct {
    40  	Backend   string                   `yaml:"backend"`
    41  	InMemory  InMemoryIndexCacheConfig `yaml:"inmemory"`
    42  	Memcached MemcachedClientConfig    `yaml:"memcached"`
    43  }
    44  
    45  func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet) {
    46  	cfg.RegisterFlagsWithPrefix(f, "blocks-storage.bucket-store.index-cache.")
    47  }
    48  
    49  func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
    50  	f.StringVar(&cfg.Backend, prefix+"backend", IndexCacheBackendDefault, fmt.Sprintf("The index cache backend type. Supported values: %s.", strings.Join(supportedIndexCacheBackends, ", ")))
    51  
    52  	cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.")
    53  	cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.")
    54  }
    55  
    56  // Validate the config.
    57  func (cfg *IndexCacheConfig) Validate() error {
    58  	if !util.StringsContain(supportedIndexCacheBackends, cfg.Backend) {
    59  		return errUnsupportedIndexCacheBackend
    60  	}
    61  
    62  	if cfg.Backend == IndexCacheBackendMemcached {
    63  		if err := cfg.Memcached.Validate(); err != nil {
    64  			return err
    65  		}
    66  	}
    67  
    68  	return nil
    69  }
    70  
    71  type InMemoryIndexCacheConfig struct {
    72  	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
    73  }
    74  
    75  func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
    76  	f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), "Maximum size in bytes of in-memory index cache used to speed up blocks index lookups (shared between all tenants).")
    77  }
    78  
    79  // NewIndexCache creates a new index cache based on the input configuration.
    80  func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) {
    81  	switch cfg.Backend {
    82  	case IndexCacheBackendInMemory:
    83  		return newInMemoryIndexCache(cfg.InMemory, logger, registerer)
    84  	case IndexCacheBackendMemcached:
    85  		return newMemcachedIndexCache(cfg.Memcached, logger, registerer)
    86  	default:
    87  		return nil, errUnsupportedIndexCacheBackend
    88  	}
    89  }
    90  
    91  func newInMemoryIndexCache(cfg InMemoryIndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) {
    92  	maxCacheSize := model.Bytes(cfg.MaxSizeBytes)
    93  
    94  	// Calculate the max item size.
    95  	maxItemSize := defaultMaxItemSize
    96  	if maxItemSize > maxCacheSize {
    97  		maxItemSize = maxCacheSize
    98  	}
    99  
   100  	return storecache.NewInMemoryIndexCacheWithConfig(logger, registerer, storecache.InMemoryIndexCacheConfig{
   101  		MaxSize:     maxCacheSize,
   102  		MaxItemSize: maxItemSize,
   103  	})
   104  }
   105  
   106  func newMemcachedIndexCache(cfg MemcachedClientConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) {
   107  	client, err := cacheutil.NewMemcachedClientWithConfig(logger, "index-cache", cfg.ToMemcachedClientConfig(), registerer)
   108  	if err != nil {
   109  		return nil, errors.Wrapf(err, "create index cache memcached client")
   110  	}
   111  
   112  	return storecache.NewMemcachedIndexCache(logger, client, registerer)
   113  }