github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/chunk/storage.go (about)

     1  package chunk
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
     8  	"github.com/pachyderm/pachyderm/src/server/pkg/storage/track"
     9  )
    10  
    11  const (
    12  	// TrackerPrefix is the prefix used when creating tracker objects for chunks
    13  	TrackerPrefix   = "chunk/"
    14  	prefix          = "chunk"
    15  	defaultChunkTTL = 30 * time.Minute
    16  )
    17  
    18  // Storage is the abstraction that manages chunk storage.
    19  type Storage struct {
    20  	objClient obj.Client
    21  	tracker   track.Tracker
    22  	mdstore   MetadataStore
    23  
    24  	defaultChunkTTL time.Duration
    25  }
    26  
    27  // NewStorage creates a new Storage.
    28  func NewStorage(objClient obj.Client, mdstore MetadataStore, tracker track.Tracker, opts ...StorageOption) *Storage {
    29  	s := &Storage{
    30  		objClient:       objClient,
    31  		mdstore:         mdstore,
    32  		defaultChunkTTL: defaultChunkTTL,
    33  		tracker:         tracker,
    34  	}
    35  	for _, opt := range opts {
    36  		opt(s)
    37  	}
    38  	return s
    39  }
    40  
    41  // NewReader creates a new Reader.
    42  func (s *Storage) NewReader(ctx context.Context, dataRefs []*DataRef) *Reader {
    43  	// using the empty string for the tmp id to disable the renewer
    44  	client := NewClient(s.objClient, s.mdstore, s.tracker, "")
    45  	return newReader(ctx, client, dataRefs)
    46  }
    47  
    48  // NewWriter creates a new Writer for a stream of bytes to be chunked.
    49  // Chunks are created based on the content, then hashed and deduplicated/uploaded to
    50  // object storage.
    51  func (s *Storage) NewWriter(ctx context.Context, tmpID string, cb WriterCallback, opts ...WriterOption) *Writer {
    52  	client := NewClient(s.objClient, s.mdstore, s.tracker, tmpID)
    53  	return newWriter(ctx, client, cb, opts...)
    54  }
    55  
    56  // List lists all of the chunks in object storage.
    57  func (s *Storage) List(ctx context.Context, cb func(string) error) error {
    58  	return s.objClient.Walk(ctx, prefix, cb)
    59  }
    60  
    61  // NewDeleter creates a deleter for use with a tracker.GC
    62  func (s *Storage) NewDeleter() track.Deleter {
    63  	return &deleter{
    64  		mdstore: s.mdstore,
    65  		objc:    s.objClient,
    66  	}
    67  }