github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/chunk/storage_client.go (about)

     1  package chunk
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"io"
     7  	"time"
     8  )
     9  
    10  var (
    11  	// ErrStorageObjectNotFound when object storage does not have requested object
    12  	ErrStorageObjectNotFound = errors.New("object not found in storage")
    13  	// ErrMethodNotImplemented when any of the storage clients do not implement a method
    14  	ErrMethodNotImplemented = errors.New("method is not implemented")
    15  )
    16  
    17  // IndexClient is a client for the storage of the index (e.g. DynamoDB or Bigtable).
    18  type IndexClient interface {
    19  	Stop()
    20  
    21  	// For the write path.
    22  	NewWriteBatch() WriteBatch
    23  	BatchWrite(context.Context, WriteBatch) error
    24  
    25  	// For the read path.
    26  	QueryPages(ctx context.Context, queries []IndexQuery, callback func(IndexQuery, ReadBatch) (shouldContinue bool)) error
    27  }
    28  
    29  // Client is for storing and retrieving chunks.
    30  type Client interface {
    31  	Stop()
    32  
    33  	PutChunks(ctx context.Context, chunks []Chunk) error
    34  	GetChunks(ctx context.Context, chunks []Chunk) ([]Chunk, error)
    35  	DeleteChunk(ctx context.Context, userID, chunkID string) error
    36  }
    37  
    38  // ObjectAndIndexClient allows optimisations where the same client handles both
    39  type ObjectAndIndexClient interface {
    40  	PutChunksAndIndex(ctx context.Context, chunks []Chunk, index WriteBatch) error
    41  }
    42  
    43  // WriteBatch represents a batch of writes.
    44  type WriteBatch interface {
    45  	Add(tableName, hashValue string, rangeValue []byte, value []byte)
    46  	Delete(tableName, hashValue string, rangeValue []byte)
    47  }
    48  
    49  // ReadBatch represents the results of a QueryPages.
    50  type ReadBatch interface {
    51  	Iterator() ReadBatchIterator
    52  }
    53  
    54  // ReadBatchIterator is an iterator over a ReadBatch.
    55  type ReadBatchIterator interface {
    56  	Next() bool
    57  	RangeValue() []byte
    58  	Value() []byte
    59  }
    60  
    61  // ObjectClient is used to store arbitrary data in Object Store (S3/GCS/Azure/...)
    62  type ObjectClient interface {
    63  	PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error
    64  	// NOTE: The consumer of GetObject should always call the Close method when it is done reading which otherwise could cause a resource leak.
    65  	GetObject(ctx context.Context, objectKey string) (io.ReadCloser, error)
    66  
    67  	// List objects with given prefix.
    68  	//
    69  	// If delimiter is empty, all objects are returned, even if they are in nested in "subdirectories".
    70  	// If delimiter is not empty, it is used to compute common prefixes ("subdirectories"),
    71  	// and objects containing delimiter in the name will not be returned in the result.
    72  	//
    73  	// For example, if the prefix is "notes/" and the delimiter is a slash (/) as in "notes/summer/july", the common prefix is "notes/summer/".
    74  	// Common prefixes will always end with passed delimiter.
    75  	//
    76  	// Keys of returned storage objects have given prefix.
    77  	List(ctx context.Context, prefix string, delimiter string) ([]StorageObject, []StorageCommonPrefix, error)
    78  	DeleteObject(ctx context.Context, objectKey string) error
    79  	Stop()
    80  }
    81  
    82  // StorageObject represents an object being stored in an Object Store
    83  type StorageObject struct {
    84  	Key        string
    85  	ModifiedAt time.Time
    86  }
    87  
    88  // StorageCommonPrefix represents a common prefix aka a synthetic directory in Object Store.
    89  // It is guaranteed to always end with delimiter passed to List method.
    90  type StorageCommonPrefix string