github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/requester/jobtransform/inline_spec_storage.go (about)

     1  package jobtransform
     2  
     3  import (
     4  	"context"
     5  
     6  	"github.com/c2h5oh/datasize"
     7  	"github.com/filecoin-project/bacalhau/pkg/model"
     8  	"github.com/filecoin-project/bacalhau/pkg/storage"
     9  	"github.com/filecoin-project/bacalhau/pkg/storage/copy"
    10  )
    11  
    12  // The maximum size that an individual inline storage spec and all inline
    13  // storage specs (respectively) can take up before being pinned to IPFS storage.
    14  const (
    15  	maximumIndividualSpec datasize.ByteSize = 5 * datasize.KB
    16  	maximumTotalSpec      datasize.ByteSize = 5 * datasize.KB
    17  )
    18  
    19  // NewInlineStoragePinner returns a job transformer that limits the inline space
    20  // taken up by inline data. It will scan a job for StorageSpec objects that
    21  // store their data inline and move any that are too large into IPFS storage. It
    22  // also limits the total size taken up by inline specs and if this value is
    23  // exceeded it will move the largest specs into IPFS.
    24  func NewInlineStoragePinner(provider storage.StorageProvider) Transformer {
    25  	return func(ctx context.Context, j *model.Job) (modified bool, err error) {
    26  		return copy.CopyOversize(
    27  			ctx,
    28  			provider,
    29  			j.Spec.AllStorageSpecs(),
    30  			model.StorageSourceInline,
    31  			model.StorageSourceIPFS,
    32  			maximumIndividualSpec,
    33  			maximumTotalSpec,
    34  		)
    35  	}
    36  }