github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/chunk/option.go (about)

     1  package chunk
     2  
     3  import (
     4  	"math"
     5  	"time"
     6  
     7  	"github.com/chmduquesne/rollinghash/buzhash64"
     8  	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
     9  )
    10  
    11  // StorageOption configures a storage.
    12  type StorageOption func(s *Storage)
    13  
    14  // WithMaxConcurrentObjects sets the maximum number of object writers (upload)
    15  // and readers (download) that can be open at a time.
    16  func WithMaxConcurrentObjects(maxDownload, maxUpload int) StorageOption {
    17  	return func(s *Storage) {
    18  		s.objClient = obj.NewLimitedClient(s.objClient, maxDownload, maxUpload)
    19  	}
    20  }
    21  
    22  // WithGCTimeout sets the default chunk ttl for this Storage instance
    23  func WithGCTimeout(timeout time.Duration) StorageOption {
    24  	return func(s *Storage) {
    25  		s.defaultChunkTTL = timeout
    26  	}
    27  }
    28  
    29  // WithObjectCache adds a cache around the currently configured object client
    30  func WithObjectCache(fastLayer obj.Client, size int) StorageOption {
    31  	return func(s *Storage) {
    32  		s.objClient = obj.NewCacheClient(s.objClient, fastLayer, size)
    33  	}
    34  }
    35  
    36  // WriterOption configures a chunk writer.
    37  type WriterOption func(w *Writer)
    38  
    39  // WithRollingHashConfig sets up the rolling hash with the passed in configuration.
    40  func WithRollingHashConfig(averageBits int, seed int64) WriterOption {
    41  	return func(w *Writer) {
    42  		w.chunkSize.avg = int(math.Pow(2, float64(averageBits)))
    43  		w.splitMask = (1 << uint64(averageBits)) - 1
    44  		w.hash = buzhash64.NewFromUint64Array(buzhash64.GenerateHashes(seed))
    45  	}
    46  }
    47  
    48  // WithMinMax sets the minimum and maximum chunk size.
    49  func WithMinMax(min, max int) WriterOption {
    50  	return func(w *Writer) {
    51  		w.chunkSize.min = min
    52  		w.chunkSize.max = max
    53  	}
    54  }
    55  
    56  // WithNoUpload sets the writer to no upload (will not upload chunks).
    57  func WithNoUpload() WriterOption {
    58  	return func(w *Writer) {
    59  		w.noUpload = true
    60  	}
    61  }