github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/storage/copy/copy.go (about)

     1  package copy
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	"github.com/c2h5oh/datasize"
     8  	"github.com/filecoin-project/bacalhau/pkg/model"
     9  	"github.com/filecoin-project/bacalhau/pkg/storage"
    10  	"github.com/filecoin-project/bacalhau/pkg/system"
    11  	"github.com/pkg/errors"
    12  	"github.com/rs/zerolog/log"
    13  	"go.uber.org/multierr"
    14  	"golang.org/x/exp/slices"
    15  )
    16  
    17  type specSize struct {
    18  	spec *model.StorageSpec
    19  	size datasize.ByteSize
    20  }
    21  
    22  // CopyOversize transfers StorageSpecs from one StorageSourceType to another in
    23  // order to fit the specs into the passed size limits.
    24  //
    25  // A spec will be transferred if it is over the passed maxSingle size. It may be
    26  // transferred if all the specs are over the passed maxTotal size, depending on
    27  // how big the other specs are (bigger specs are transferred first).
    28  //
    29  // The specs will be updated in place to contain the location of the new data.
    30  // If any specs are not of the passed srcType, they are ignored.
    31  //
    32  // Passing 0 as either limit will cause all specs to be transferred.
    33  func CopyOversize(
    34  	ctx context.Context,
    35  	provider storage.StorageProvider,
    36  	specs []*model.StorageSpec,
    37  	srcType, dstType model.StorageSourceType,
    38  	maxSingle, maxTotal datasize.ByteSize,
    39  ) (modified bool, err error) {
    40  	srcStorage, err := provider.Get(ctx, srcType)
    41  	if err != nil {
    42  		err = errors.Wrapf(err, "failed to get %s storage provider", srcType)
    43  		return
    44  	}
    45  
    46  	specsizes := make([]specSize, 0, len(specs))
    47  	for _, spec := range specs {
    48  		if spec.StorageSource != srcType {
    49  			continue
    50  		}
    51  
    52  		size, rerr := srcStorage.GetVolumeSize(ctx, *spec)
    53  		if rerr != nil {
    54  			err = errors.Wrapf(rerr, "failed to read spec %v", spec)
    55  			return
    56  		}
    57  		specsizes = append(specsizes, specSize{spec: spec, size: datasize.ByteSize(size)})
    58  	}
    59  
    60  	slices.SortFunc(specsizes, func(a, b specSize) bool {
    61  		return a.size < b.size
    62  	})
    63  
    64  	remainingSpace := maxTotal
    65  	for _, spec := range specsizes {
    66  		exactFit := spec.size == remainingSpace
    67  		remainingSpace -= system.Min(spec.size, remainingSpace)
    68  		if (!exactFit && remainingSpace <= 0) || maxTotal == 0 || spec.size > maxSingle {
    69  			newSpec, rerr := Copy(ctx, provider, *spec.spec, dstType)
    70  			if rerr != nil {
    71  				return modified, rerr
    72  			}
    73  
    74  			*spec.spec = newSpec
    75  			log.Ctx(ctx).Debug().
    76  				Str("Spec", fmt.Sprint(newSpec)).
    77  				Stringer("OldSource", srcType).
    78  				Msg("Replaced spec")
    79  			modified = true
    80  		}
    81  	}
    82  
    83  	return modified, err
    84  }
    85  
    86  // Copy transfers data described by the passed StorageSpec into the destination
    87  // type, and returns a new StorageSpec for the data in its new location.
    88  func Copy(
    89  	ctx context.Context,
    90  	provider storage.StorageProvider,
    91  	spec model.StorageSpec,
    92  	destination model.StorageSourceType,
    93  ) (model.StorageSpec, error) {
    94  	srcStorage, srcErr := provider.Get(ctx, spec.StorageSource)
    95  	dstStorage, dstErr := provider.Get(ctx, destination)
    96  	err := multierr.Append(srcErr, dstErr)
    97  	if err != nil {
    98  		return model.StorageSpec{}, err
    99  	}
   100  
   101  	volume, err := srcStorage.PrepareStorage(ctx, spec)
   102  	if err != nil {
   103  		err = errors.Wrapf(err, "failed to prepare %s spec", spec.StorageSource)
   104  		return model.StorageSpec{}, err
   105  	}
   106  	defer srcStorage.CleanupStorage(ctx, spec, volume) //nolint:errcheck
   107  
   108  	var newSpec model.StorageSpec
   109  	newSpec, err = dstStorage.Upload(ctx, volume.Source)
   110  	if err != nil {
   111  		err = errors.Wrapf(err, "failed to save %s spec to %s", spec.StorageSource, destination)
   112  	}
   113  	return newSpec, err
   114  }