github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/storage/bucket/client.go (about)

     1  package bucket
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"flag"
     7  	"fmt"
     8  	"strings"
     9  
    10  	"github.com/go-kit/log"
    11  	"github.com/prometheus/client_golang/prometheus"
    12  	"github.com/thanos-io/thanos/pkg/objstore"
    13  
    14  	"github.com/cortexproject/cortex/pkg/storage/bucket/azure"
    15  	"github.com/cortexproject/cortex/pkg/storage/bucket/filesystem"
    16  	"github.com/cortexproject/cortex/pkg/storage/bucket/gcs"
    17  	"github.com/cortexproject/cortex/pkg/storage/bucket/s3"
    18  	"github.com/cortexproject/cortex/pkg/storage/bucket/swift"
    19  	"github.com/cortexproject/cortex/pkg/util"
    20  )
    21  
    22  const (
    23  	// S3 is the value for the S3 storage backend.
    24  	S3 = "s3"
    25  
    26  	// GCS is the value for the GCS storage backend.
    27  	GCS = "gcs"
    28  
    29  	// Azure is the value for the Azure storage backend.
    30  	Azure = "azure"
    31  
    32  	// Swift is the value for the Openstack Swift storage backend.
    33  	Swift = "swift"
    34  
    35  	// Filesystem is the value for the filesystem storage backend.
    36  	Filesystem = "filesystem"
    37  )
    38  
    39  var (
    40  	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}
    41  
    42  	ErrUnsupportedStorageBackend = errors.New("unsupported storage backend")
    43  )
    44  
    45  // Config holds configuration for accessing long-term storage.
    46  type Config struct {
    47  	Backend string `yaml:"backend"`
    48  	// Backends
    49  	S3         s3.Config         `yaml:"s3"`
    50  	GCS        gcs.Config        `yaml:"gcs"`
    51  	Azure      azure.Config      `yaml:"azure"`
    52  	Swift      swift.Config      `yaml:"swift"`
    53  	Filesystem filesystem.Config `yaml:"filesystem"`
    54  
    55  	// Not used internally, meant to allow callers to wrap Buckets
    56  	// created using this config
    57  	Middlewares []func(objstore.Bucket) (objstore.Bucket, error) `yaml:"-"`
    58  
    59  	// Used to inject additional backends into the config. Allows for this config to
    60  	// be embedded in multiple contexts and support non-object storage based backends.
    61  	ExtraBackends []string `yaml:"-"`
    62  }
    63  
    64  // Returns the supportedBackends for the package and any custom backends injected into the config.
    65  func (cfg *Config) supportedBackends() []string {
    66  	return append(SupportedBackends, cfg.ExtraBackends...)
    67  }
    68  
    69  // RegisterFlags registers the backend storage config.
    70  func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
    71  	cfg.RegisterFlagsWithPrefix("", f)
    72  }
    73  
    74  func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
    75  	cfg.S3.RegisterFlagsWithPrefix(prefix, f)
    76  	cfg.GCS.RegisterFlagsWithPrefix(prefix, f)
    77  	cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
    78  	cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
    79  	cfg.Filesystem.RegisterFlagsWithPrefix(prefix, f)
    80  
    81  	f.StringVar(&cfg.Backend, prefix+"backend", "s3", fmt.Sprintf("Backend storage to use. Supported backends are: %s.", strings.Join(cfg.supportedBackends(), ", ")))
    82  }
    83  
    84  func (cfg *Config) Validate() error {
    85  	if !util.StringsContain(cfg.supportedBackends(), cfg.Backend) {
    86  		return ErrUnsupportedStorageBackend
    87  	}
    88  
    89  	if cfg.Backend == S3 {
    90  		if err := cfg.S3.Validate(); err != nil {
    91  			return err
    92  		}
    93  	}
    94  
    95  	return nil
    96  }
    97  
    98  // NewClient creates a new bucket client based on the configured backend
    99  func NewClient(ctx context.Context, cfg Config, name string, logger log.Logger, reg prometheus.Registerer) (client objstore.Bucket, err error) {
   100  	switch cfg.Backend {
   101  	case S3:
   102  		client, err = s3.NewBucketClient(cfg.S3, name, logger)
   103  	case GCS:
   104  		client, err = gcs.NewBucketClient(ctx, cfg.GCS, name, logger)
   105  	case Azure:
   106  		client, err = azure.NewBucketClient(cfg.Azure, name, logger)
   107  	case Swift:
   108  		client, err = swift.NewBucketClient(cfg.Swift, name, logger)
   109  	case Filesystem:
   110  		client, err = filesystem.NewBucketClient(cfg.Filesystem)
   111  	default:
   112  		return nil, ErrUnsupportedStorageBackend
   113  	}
   114  
   115  	if err != nil {
   116  		return nil, err
   117  	}
   118  
   119  	client = objstore.NewTracingBucket(bucketWithMetrics(client, name, reg))
   120  
   121  	// Wrap the client with any provided middleware
   122  	for _, wrap := range cfg.Middlewares {
   123  		client, err = wrap(client)
   124  		if err != nil {
   125  			return nil, err
   126  		}
   127  	}
   128  
   129  	return client, nil
   130  }
   131  
   132  func bucketWithMetrics(bucketClient objstore.Bucket, name string, reg prometheus.Registerer) objstore.Bucket {
   133  	if reg == nil {
   134  		return bucketClient
   135  	}
   136  
   137  	return objstore.BucketWithMetrics(
   138  		"", // bucket label value
   139  		bucketClient,
   140  		prometheus.WrapRegistererWith(prometheus.Labels{"component": name}, reg))
   141  }