github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/fileset/option.go (about)

     1  package fileset
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"
     7  	"github.com/pachyderm/pachyderm/src/server/pkg/storage/renew"
     8  	"golang.org/x/sync/semaphore"
     9  )
    10  
    11  // StorageOption configures a storage.
    12  type StorageOption func(*Storage)
    13  
    14  // WithMemoryThreshold sets the memory threshold that must
    15  // be met before a file set part is serialized (excluding close).
    16  func WithMemoryThreshold(threshold int64) StorageOption {
    17  	return func(s *Storage) {
    18  		s.memThreshold = threshold
    19  	}
    20  }
    21  
    22  // WithShardThreshold sets the size threshold that must
    23  // be met before a shard is created by the shard function.
    24  func WithShardThreshold(threshold int64) StorageOption {
    25  	return func(s *Storage) {
    26  		s.shardThreshold = threshold
    27  	}
    28  }
    29  
    30  // WithLevelZeroSize sets the size for level zero in the compacted
    31  // representation of a file set.
    32  func WithLevelZeroSize(size int64) StorageOption {
    33  	return func(s *Storage) {
    34  		s.levelZeroSize = size
    35  	}
    36  }
    37  
    38  // WithLevelSizeBase sets the base of the exponential growth function
    39  // for level sizes in the compacted representation of a file set.
    40  func WithLevelSizeBase(base int) StorageOption {
    41  	return func(s *Storage) {
    42  		s.levelSizeBase = base
    43  	}
    44  }
    45  
    46  // WithMaxOpenFileSets sets the maximum number of filesets that will be open
    47  // (potentially buffered in memory) at a time.
    48  func WithMaxOpenFileSets(max int) StorageOption {
    49  	return func(s *Storage) {
    50  		s.filesetSem = semaphore.NewWeighted(int64(max))
    51  	}
    52  }
    53  
    54  // UnorderedWriterOption configures an UnorderedWriter.
    55  type UnorderedWriterOption func(*UnorderedWriter)
    56  
    57  // WithRenewal configures the UnorderedWriter to renew subfileset paths
    58  // with the provided renewer.
    59  func WithRenewal(ttl time.Duration, r *renew.StringSet) UnorderedWriterOption {
    60  	return func(uw *UnorderedWriter) {
    61  		uw.ttl = ttl
    62  		uw.renewer = r
    63  	}
    64  }
    65  
    66  // WriterOption configures a file set writer.
    67  type WriterOption func(w *Writer)
    68  
    69  // WithNoUpload sets the writer to no upload (will not upload chunks).
    70  func WithNoUpload() WriterOption {
    71  	return func(w *Writer) {
    72  		w.noUpload = true
    73  	}
    74  }
    75  
    76  // WithIndexCallback sets a function to be called after each index is written.
    77  // If WithNoUpload is set, the function is called after the index would have been written.
    78  func WithIndexCallback(cb func(*index.Index) error) WriterOption {
    79  	return func(w *Writer) {
    80  		w.indexFunc = cb
    81  	}
    82  }
    83  
    84  // WithTTL sets the ttl for the fileset
    85  func WithTTL(ttl time.Duration) WriterOption {
    86  	return func(w *Writer) {
    87  		w.ttl = ttl
    88  	}
    89  }