github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/client/openstack/swift_object_client.go (about)

     1  package openstack
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"flag"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"net/http"
    11  	"time"
    12  
    13  	"github.com/ncw/swift"
    14  	"github.com/pkg/errors"
    15  	"github.com/prometheus/client_golang/prometheus"
    16  
    17  	bucket_swift "github.com/grafana/loki/pkg/storage/bucket/swift"
    18  	"github.com/grafana/loki/pkg/storage/chunk/client"
    19  	"github.com/grafana/loki/pkg/storage/chunk/client/hedging"
    20  	"github.com/grafana/loki/pkg/util/log"
    21  )
    22  
    23  var defaultTransport http.RoundTripper = &http.Transport{
    24  	Proxy:                 http.ProxyFromEnvironment,
    25  	MaxIdleConnsPerHost:   200,
    26  	MaxIdleConns:          200,
    27  	ExpectContinueTimeout: 5 * time.Second,
    28  }
    29  
    30  type SwiftObjectClient struct {
    31  	conn        *swift.Connection
    32  	hedgingConn *swift.Connection
    33  	cfg         SwiftConfig
    34  }
    35  
    36  // SwiftConfig is config for the Swift Chunk Client.
    37  type SwiftConfig struct {
    38  	bucket_swift.Config `yaml:",inline"`
    39  }
    40  
    41  // RegisterFlags registers flags.
    42  func (cfg *SwiftConfig) RegisterFlags(f *flag.FlagSet) {
    43  	cfg.RegisterFlagsWithPrefix("", f)
    44  }
    45  
    46  // Validate config and returns error on failure
    47  func (cfg *SwiftConfig) Validate() error {
    48  	return nil
    49  }
    50  
    51  // RegisterFlagsWithPrefix registers flags with prefix.
    52  func (cfg *SwiftConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
    53  	cfg.Config.RegisterFlagsWithPrefix(prefix, f)
    54  }
    55  
    56  // NewSwiftObjectClient makes a new chunk.Client that writes chunks to OpenStack Swift.
    57  func NewSwiftObjectClient(cfg SwiftConfig, hedgingCfg hedging.Config) (*SwiftObjectClient, error) {
    58  	log.WarnExperimentalUse("OpenStack Swift Storage", log.Logger)
    59  
    60  	c, err := createConnection(cfg, hedgingCfg, false)
    61  	if err != nil {
    62  		return nil, err
    63  	}
    64  	// Ensure the container is created, no error is returned if it already exists.
    65  	if err := c.ContainerCreate(cfg.ContainerName, nil); err != nil {
    66  		return nil, err
    67  	}
    68  	hedging, err := createConnection(cfg, hedgingCfg, true)
    69  	if err != nil {
    70  		return nil, err
    71  	}
    72  	return &SwiftObjectClient{
    73  		conn:        c,
    74  		hedgingConn: hedging,
    75  		cfg:         cfg,
    76  	}, nil
    77  }
    78  
    79  func createConnection(cfg SwiftConfig, hedgingCfg hedging.Config, hedging bool) (*swift.Connection, error) {
    80  	// Create a connection
    81  	c := &swift.Connection{
    82  		AuthVersion:    cfg.AuthVersion,
    83  		AuthUrl:        cfg.AuthURL,
    84  		ApiKey:         cfg.Password,
    85  		UserName:       cfg.Username,
    86  		UserId:         cfg.UserID,
    87  		Retries:        cfg.MaxRetries,
    88  		ConnectTimeout: cfg.ConnectTimeout,
    89  		Timeout:        cfg.RequestTimeout,
    90  		TenantId:       cfg.ProjectID,
    91  		Tenant:         cfg.ProjectName,
    92  		TenantDomain:   cfg.ProjectDomainName,
    93  		TenantDomainId: cfg.ProjectDomainID,
    94  		Domain:         cfg.DomainName,
    95  		DomainId:       cfg.DomainID,
    96  		Region:         cfg.RegionName,
    97  		Transport:      defaultTransport,
    98  	}
    99  
   100  	switch {
   101  	case cfg.UserDomainName != "":
   102  		c.Domain = cfg.UserDomainName
   103  	case cfg.UserDomainID != "":
   104  		c.DomainId = cfg.UserDomainID
   105  	}
   106  	if hedging {
   107  		var err error
   108  		c.Transport, err = hedgingCfg.RoundTripperWithRegisterer(c.Transport, prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer))
   109  		if err != nil {
   110  			return nil, err
   111  		}
   112  	}
   113  
   114  	err := c.Authenticate()
   115  	if err != nil {
   116  		return nil, err
   117  	}
   118  
   119  	return c, nil
   120  }
   121  
   122  func (s *SwiftObjectClient) Stop() {
   123  	s.conn.UnAuthenticate()
   124  	s.hedgingConn.UnAuthenticate()
   125  }
   126  
   127  // GetObject returns a reader and the size for the specified object key from the configured swift container.
   128  func (s *SwiftObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) {
   129  	var buf bytes.Buffer
   130  	_, err := s.hedgingConn.ObjectGet(s.cfg.ContainerName, objectKey, &buf, false, nil)
   131  	if err != nil {
   132  		return nil, 0, err
   133  	}
   134  
   135  	return ioutil.NopCloser(&buf), int64(buf.Len()), nil
   136  }
   137  
   138  // PutObject puts the specified bytes into the configured Swift container at the provided key
   139  func (s *SwiftObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error {
   140  	_, err := s.conn.ObjectPut(s.cfg.ContainerName, objectKey, object, false, "", "", nil)
   141  	return err
   142  }
   143  
   144  // List only objects from the store non-recursively
   145  func (s *SwiftObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error) {
   146  	if len(delimiter) > 1 {
   147  		return nil, nil, fmt.Errorf("delimiter must be a single character but was %s", delimiter)
   148  	}
   149  
   150  	opts := &swift.ObjectsOpts{
   151  		Prefix: prefix,
   152  	}
   153  	if len(delimiter) > 0 {
   154  		opts.Delimiter = []rune(delimiter)[0]
   155  	}
   156  
   157  	objs, err := s.conn.Objects(s.cfg.ContainerName, opts)
   158  	if err != nil {
   159  		return nil, nil, err
   160  	}
   161  
   162  	var storageObjects []client.StorageObject
   163  	var storagePrefixes []client.StorageCommonPrefix
   164  
   165  	for _, obj := range objs {
   166  		// based on the docs when subdir is set, it means it's a pseudo directory.
   167  		// see https://docs.openstack.org/swift/latest/api/pseudo-hierarchical-folders-directories.html
   168  		if obj.SubDir != "" {
   169  			storagePrefixes = append(storagePrefixes, client.StorageCommonPrefix(obj.SubDir))
   170  			continue
   171  		}
   172  
   173  		storageObjects = append(storageObjects, client.StorageObject{
   174  			Key:        obj.Name,
   175  			ModifiedAt: obj.LastModified,
   176  		})
   177  	}
   178  
   179  	return storageObjects, storagePrefixes, nil
   180  }
   181  
   182  // DeleteObject deletes the specified object key from the configured Swift container.
   183  func (s *SwiftObjectClient) DeleteObject(ctx context.Context, objectKey string) error {
   184  	return s.conn.ObjectDelete(s.cfg.ContainerName, objectKey)
   185  }
   186  
   187  // IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.
   188  func (s *SwiftObjectClient) IsObjectNotFoundErr(err error) bool {
   189  	return errors.Is(err, swift.ObjectNotFound)
   190  }