github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/client/openstack/swift_object_client.go (about) 1 package openstack 2 3 import ( 4 "bytes" 5 "context" 6 "flag" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "net/http" 11 "time" 12 13 "github.com/ncw/swift" 14 "github.com/pkg/errors" 15 "github.com/prometheus/client_golang/prometheus" 16 17 bucket_swift "github.com/grafana/loki/pkg/storage/bucket/swift" 18 "github.com/grafana/loki/pkg/storage/chunk/client" 19 "github.com/grafana/loki/pkg/storage/chunk/client/hedging" 20 "github.com/grafana/loki/pkg/util/log" 21 ) 22 23 var defaultTransport http.RoundTripper = &http.Transport{ 24 Proxy: http.ProxyFromEnvironment, 25 MaxIdleConnsPerHost: 200, 26 MaxIdleConns: 200, 27 ExpectContinueTimeout: 5 * time.Second, 28 } 29 30 type SwiftObjectClient struct { 31 conn *swift.Connection 32 hedgingConn *swift.Connection 33 cfg SwiftConfig 34 } 35 36 // SwiftConfig is config for the Swift Chunk Client. 37 type SwiftConfig struct { 38 bucket_swift.Config `yaml:",inline"` 39 } 40 41 // RegisterFlags registers flags. 42 func (cfg *SwiftConfig) RegisterFlags(f *flag.FlagSet) { 43 cfg.RegisterFlagsWithPrefix("", f) 44 } 45 46 // Validate config and returns error on failure 47 func (cfg *SwiftConfig) Validate() error { 48 return nil 49 } 50 51 // RegisterFlagsWithPrefix registers flags with prefix. 52 func (cfg *SwiftConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { 53 cfg.Config.RegisterFlagsWithPrefix(prefix, f) 54 } 55 56 // NewSwiftObjectClient makes a new chunk.Client that writes chunks to OpenStack Swift. 57 func NewSwiftObjectClient(cfg SwiftConfig, hedgingCfg hedging.Config) (*SwiftObjectClient, error) { 58 log.WarnExperimentalUse("OpenStack Swift Storage", log.Logger) 59 60 c, err := createConnection(cfg, hedgingCfg, false) 61 if err != nil { 62 return nil, err 63 } 64 // Ensure the container is created, no error is returned if it already exists. 65 if err := c.ContainerCreate(cfg.ContainerName, nil); err != nil { 66 return nil, err 67 } 68 hedging, err := createConnection(cfg, hedgingCfg, true) 69 if err != nil { 70 return nil, err 71 } 72 return &SwiftObjectClient{ 73 conn: c, 74 hedgingConn: hedging, 75 cfg: cfg, 76 }, nil 77 } 78 79 func createConnection(cfg SwiftConfig, hedgingCfg hedging.Config, hedging bool) (*swift.Connection, error) { 80 // Create a connection 81 c := &swift.Connection{ 82 AuthVersion: cfg.AuthVersion, 83 AuthUrl: cfg.AuthURL, 84 ApiKey: cfg.Password, 85 UserName: cfg.Username, 86 UserId: cfg.UserID, 87 Retries: cfg.MaxRetries, 88 ConnectTimeout: cfg.ConnectTimeout, 89 Timeout: cfg.RequestTimeout, 90 TenantId: cfg.ProjectID, 91 Tenant: cfg.ProjectName, 92 TenantDomain: cfg.ProjectDomainName, 93 TenantDomainId: cfg.ProjectDomainID, 94 Domain: cfg.DomainName, 95 DomainId: cfg.DomainID, 96 Region: cfg.RegionName, 97 Transport: defaultTransport, 98 } 99 100 switch { 101 case cfg.UserDomainName != "": 102 c.Domain = cfg.UserDomainName 103 case cfg.UserDomainID != "": 104 c.DomainId = cfg.UserDomainID 105 } 106 if hedging { 107 var err error 108 c.Transport, err = hedgingCfg.RoundTripperWithRegisterer(c.Transport, prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer)) 109 if err != nil { 110 return nil, err 111 } 112 } 113 114 err := c.Authenticate() 115 if err != nil { 116 return nil, err 117 } 118 119 return c, nil 120 } 121 122 func (s *SwiftObjectClient) Stop() { 123 s.conn.UnAuthenticate() 124 s.hedgingConn.UnAuthenticate() 125 } 126 127 // GetObject returns a reader and the size for the specified object key from the configured swift container. 128 func (s *SwiftObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { 129 var buf bytes.Buffer 130 _, err := s.hedgingConn.ObjectGet(s.cfg.ContainerName, objectKey, &buf, false, nil) 131 if err != nil { 132 return nil, 0, err 133 } 134 135 return ioutil.NopCloser(&buf), int64(buf.Len()), nil 136 } 137 138 // PutObject puts the specified bytes into the configured Swift container at the provided key 139 func (s *SwiftObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error { 140 _, err := s.conn.ObjectPut(s.cfg.ContainerName, objectKey, object, false, "", "", nil) 141 return err 142 } 143 144 // List only objects from the store non-recursively 145 func (s *SwiftObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error) { 146 if len(delimiter) > 1 { 147 return nil, nil, fmt.Errorf("delimiter must be a single character but was %s", delimiter) 148 } 149 150 opts := &swift.ObjectsOpts{ 151 Prefix: prefix, 152 } 153 if len(delimiter) > 0 { 154 opts.Delimiter = []rune(delimiter)[0] 155 } 156 157 objs, err := s.conn.Objects(s.cfg.ContainerName, opts) 158 if err != nil { 159 return nil, nil, err 160 } 161 162 var storageObjects []client.StorageObject 163 var storagePrefixes []client.StorageCommonPrefix 164 165 for _, obj := range objs { 166 // based on the docs when subdir is set, it means it's a pseudo directory. 167 // see https://docs.openstack.org/swift/latest/api/pseudo-hierarchical-folders-directories.html 168 if obj.SubDir != "" { 169 storagePrefixes = append(storagePrefixes, client.StorageCommonPrefix(obj.SubDir)) 170 continue 171 } 172 173 storageObjects = append(storageObjects, client.StorageObject{ 174 Key: obj.Name, 175 ModifiedAt: obj.LastModified, 176 }) 177 } 178 179 return storageObjects, storagePrefixes, nil 180 } 181 182 // DeleteObject deletes the specified object key from the configured Swift container. 183 func (s *SwiftObjectClient) DeleteObject(ctx context.Context, objectKey string) error { 184 return s.conn.ObjectDelete(s.cfg.ContainerName, objectKey) 185 } 186 187 // IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations. 188 func (s *SwiftObjectClient) IsObjectNotFoundErr(err error) bool { 189 return errors.Is(err, swift.ObjectNotFound) 190 }