github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/client/object_client.go (about) 1 package client 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/base64" 7 "io" 8 "strings" 9 "time" 10 11 "github.com/pkg/errors" 12 13 "github.com/grafana/loki/pkg/storage/chunk" 14 "github.com/grafana/loki/pkg/storage/chunk/client/util" 15 "github.com/grafana/loki/pkg/storage/config" 16 ) 17 18 // ObjectClient is used to store arbitrary data in Object Store (S3/GCS/Azure/...) 19 type ObjectClient interface { 20 PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error 21 // NOTE: The consumer of GetObject should always call the Close method when it is done reading which otherwise could cause a resource leak. 22 GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) 23 24 // List objects with given prefix. 25 // 26 // If delimiter is empty, all objects are returned, even if they are in nested in "subdirectories". 27 // If delimiter is not empty, it is used to compute common prefixes ("subdirectories"), 28 // and objects containing delimiter in the name will not be returned in the result. 29 // 30 // For example, if the prefix is "notes/" and the delimiter is a slash (/) as in "notes/summer/july", the common prefix is "notes/summer/". 31 // Common prefixes will always end with passed delimiter. 32 // 33 // Keys of returned storage objects have given prefix. 34 List(ctx context.Context, prefix string, delimiter string) ([]StorageObject, []StorageCommonPrefix, error) 35 DeleteObject(ctx context.Context, objectKey string) error 36 IsObjectNotFoundErr(err error) bool 37 Stop() 38 } 39 40 // StorageObject represents an object being stored in an Object Store 41 type StorageObject struct { 42 Key string 43 ModifiedAt time.Time 44 } 45 46 // StorageCommonPrefix represents a common prefix aka a synthetic directory in Object Store. 47 // It is guaranteed to always end with delimiter passed to List method. 48 type StorageCommonPrefix string 49 50 // KeyEncoder is used to encode chunk keys before writing/retrieving chunks 51 // from the underlying ObjectClient 52 // Schema/Chunk are passed as arguments to allow this to improve over revisions 53 type KeyEncoder func(schema config.SchemaConfig, chk chunk.Chunk) string 54 55 // base64Encoder is used to encode chunk keys in base64 before storing/retrieving 56 // them from the ObjectClient 57 var base64Encoder = func(key string) string { 58 return base64.StdEncoding.EncodeToString([]byte(key)) 59 } 60 61 var FSEncoder = func(schema config.SchemaConfig, chk chunk.Chunk) string { 62 // Filesystem encoder pre-v12 encodes the chunk as one base64 string. 63 // This has the downside of making them opaque and storing all chunks in a single 64 // directory, hurting performance at scale and discoverability. 65 // Post v12, we respect the directory structure imposed by chunk keys. 66 key := schema.ExternalKey(chk.ChunkRef) 67 if schema.VersionForChunk(chk.ChunkRef) > 11 { 68 split := strings.LastIndexByte(key, '/') 69 encodedTail := base64Encoder(key[split+1:]) 70 return strings.Join([]string{key[:split], encodedTail}, "/") 71 72 } 73 return base64Encoder(key) 74 } 75 76 const defaultMaxParallel = 150 77 78 // client is used to store chunks in object store backends 79 type client struct { 80 store ObjectClient 81 keyEncoder KeyEncoder 82 getChunkMaxParallel int 83 schema config.SchemaConfig 84 } 85 86 // NewClient wraps the provided ObjectClient with a chunk.Client implementation 87 func NewClient(store ObjectClient, encoder KeyEncoder, schema config.SchemaConfig) Client { 88 return NewClientWithMaxParallel(store, encoder, defaultMaxParallel, schema) 89 } 90 91 func NewClientWithMaxParallel(store ObjectClient, encoder KeyEncoder, maxParallel int, schema config.SchemaConfig) Client { 92 return &client{ 93 store: store, 94 keyEncoder: encoder, 95 getChunkMaxParallel: maxParallel, 96 schema: schema, 97 } 98 } 99 100 // Stop shuts down the object store and any underlying clients 101 func (o *client) Stop() { 102 o.store.Stop() 103 } 104 105 // PutChunks stores the provided chunks in the configured backend. If multiple errors are 106 // returned, the last one sequentially will be propagated up. 107 func (o *client) PutChunks(ctx context.Context, chunks []chunk.Chunk) error { 108 var ( 109 chunkKeys []string 110 chunkBufs [][]byte 111 ) 112 113 for i := range chunks { 114 buf, err := chunks[i].Encoded() 115 if err != nil { 116 return err 117 } 118 119 var key string 120 if o.keyEncoder != nil { 121 key = o.keyEncoder(o.schema, chunks[i]) 122 } else { 123 key = o.schema.ExternalKey(chunks[i].ChunkRef) 124 } 125 126 chunkKeys = append(chunkKeys, key) 127 chunkBufs = append(chunkBufs, buf) 128 } 129 130 incomingErrors := make(chan error) 131 for i := range chunkBufs { 132 go func(i int) { 133 incomingErrors <- o.store.PutObject(ctx, chunkKeys[i], bytes.NewReader(chunkBufs[i])) 134 }(i) 135 } 136 137 var lastErr error 138 for range chunkKeys { 139 err := <-incomingErrors 140 if err != nil { 141 lastErr = err 142 } 143 } 144 return lastErr 145 } 146 147 // GetChunks retrieves the specified chunks from the configured backend 148 func (o *client) GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) { 149 getChunkMaxParallel := o.getChunkMaxParallel 150 if getChunkMaxParallel == 0 { 151 getChunkMaxParallel = defaultMaxParallel 152 } 153 return util.GetParallelChunks(ctx, getChunkMaxParallel, chunks, o.getChunk) 154 } 155 156 func (o *client) getChunk(ctx context.Context, decodeContext *chunk.DecodeContext, c chunk.Chunk) (chunk.Chunk, error) { 157 if ctx.Err() != nil { 158 return chunk.Chunk{}, ctx.Err() 159 } 160 161 key := o.schema.ExternalKey(c.ChunkRef) 162 if o.keyEncoder != nil { 163 key = o.keyEncoder(o.schema, c) 164 } 165 166 readCloser, size, err := o.store.GetObject(ctx, key) 167 if err != nil { 168 return chunk.Chunk{}, errors.WithStack(err) 169 } 170 171 defer readCloser.Close() 172 173 // adds bytes.MinRead to avoid allocations when the size is known. 174 // This is because ReadFrom reads bytes.MinRead by bytes.MinRead. 175 buf := bytes.NewBuffer(make([]byte, 0, size+bytes.MinRead)) 176 _, err = buf.ReadFrom(readCloser) 177 if err != nil { 178 return chunk.Chunk{}, errors.WithStack(err) 179 } 180 181 if err := c.Decode(decodeContext, buf.Bytes()); err != nil { 182 return chunk.Chunk{}, errors.WithStack(err) 183 } 184 return c, nil 185 } 186 187 // GetChunks retrieves the specified chunks from the configured backend 188 func (o *client) DeleteChunk(ctx context.Context, userID, chunkID string) error { 189 key := chunkID 190 if o.keyEncoder != nil { 191 c, err := chunk.ParseExternalKey(userID, key) 192 if err != nil { 193 return err 194 } 195 key = o.keyEncoder(o.schema, c) 196 } 197 return o.store.DeleteObject(ctx, key) 198 } 199 200 func (o *client) IsChunkNotFoundErr(err error) bool { 201 return o.store.IsObjectNotFoundErr(err) 202 }