github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/storage/copy/copy_test.go (about)

     1  package copy
     2  
     3  import (
     4  	"context"
     5  	"strings"
     6  	"testing"
     7  
     8  	"github.com/c2h5oh/datasize"
     9  	"github.com/filecoin-project/bacalhau/pkg/model"
    10  	"github.com/filecoin-project/bacalhau/pkg/storage"
    11  	"github.com/filecoin-project/bacalhau/pkg/storage/noop"
    12  	"github.com/filecoin-project/bacalhau/pkg/system"
    13  	"github.com/stretchr/testify/require"
    14  )
    15  
    16  type copyOversizeTestCase struct {
    17  	name     string
    18  	specs    []*model.StorageSpec
    19  	modified bool
    20  }
    21  
    22  const (
    23  	maxSingle datasize.ByteSize       = 10 * datasize.B
    24  	maxTotal  datasize.ByteSize       = 10 * datasize.B
    25  	srcType   model.StorageSourceType = model.StorageSourceInline
    26  	dstType   model.StorageSourceType = model.StorageSourceIPFS
    27  )
    28  
    29  var copyOversizeTestCases = []copyOversizeTestCase{
    30  	{
    31  		name:     "non specs are unchanged",
    32  		specs:    []*model.StorageSpec{{StorageSource: dstType}},
    33  		modified: false,
    34  	},
    35  	{
    36  		name: "small specs are unchanged",
    37  		specs: []*model.StorageSpec{{
    38  			StorageSource: srcType,
    39  			URL:           strings.Repeat("a", int(maxSingle)),
    40  		}},
    41  		modified: false,
    42  	},
    43  	{
    44  		name: "large specs are changed",
    45  		specs: []*model.StorageSpec{{
    46  			StorageSource: srcType,
    47  			URL:           strings.Repeat("a", int(maxSingle)+1),
    48  		}},
    49  		modified: true,
    50  	},
    51  	{
    52  		name: "multiple small specs below threshold are unchanged",
    53  		specs: []*model.StorageSpec{{
    54  			StorageSource: srcType,
    55  			URL:           strings.Repeat("a", int(maxTotal/2)),
    56  		}, {
    57  			StorageSource: srcType,
    58  			URL:           strings.Repeat("a", int(maxTotal/2)),
    59  		}},
    60  		modified: false,
    61  	},
    62  	{
    63  		name: "multiple small specs above threshold are changed",
    64  		specs: []*model.StorageSpec{{
    65  			StorageSource: srcType,
    66  			URL:           strings.Repeat("a", int(maxTotal/2)+1),
    67  		}, {
    68  			StorageSource: srcType,
    69  			URL:           strings.Repeat("a", int(maxTotal/2)),
    70  		}},
    71  		modified: true,
    72  	},
    73  }
    74  
    75  func preserveSlice[T any](slice []*T) []T {
    76  	originals := make([]T, len(slice))
    77  	for i, value := range slice {
    78  		originals[i] = *value
    79  	}
    80  	return originals
    81  }
    82  
    83  func TestCopyOversize(t *testing.T) {
    84  	for _, testCase := range copyOversizeTestCases {
    85  		t.Run(testCase.name, func(t *testing.T) {
    86  			cm := system.NewCleanupManager()
    87  			t.Cleanup(func() {
    88  				cm.Cleanup(context.Background())
    89  			})
    90  
    91  			originals := preserveSlice(testCase.specs)
    92  
    93  			didUpload := false
    94  			noopStorage, err := noop.NewNoopStorageWithConfig(context.Background(), cm, noop.StorageConfig{
    95  				ExternalHooks: noop.StorageConfigExternalHooks{
    96  					GetVolumeSize: func(ctx context.Context, volume model.StorageSpec) (uint64, error) {
    97  						return uint64(len(volume.URL)), nil
    98  					},
    99  					Upload: func(ctx context.Context, localPath string) (model.StorageSpec, error) {
   100  						didUpload = true
   101  						return model.StorageSpec{StorageSource: dstType}, nil
   102  					},
   103  				},
   104  			})
   105  			require.NoError(t, err)
   106  
   107  			provider := model.NewNoopProvider[model.StorageSourceType, storage.Storage](noopStorage)
   108  			modified, err := CopyOversize(
   109  				context.Background(),
   110  				provider,
   111  				testCase.specs,
   112  				srcType,
   113  				dstType,
   114  				maxSingle,
   115  				maxTotal,
   116  			)
   117  			require.NoError(t, err)
   118  			require.Equal(t, modified, testCase.modified)
   119  			require.Equal(t, modified, didUpload)
   120  
   121  			newSpecs := preserveSlice(testCase.specs)
   122  			if modified {
   123  				require.NotSubset(t, originals, newSpecs)
   124  				require.NotSubset(t, newSpecs, originals)
   125  			} else {
   126  				require.Subset(t, originals, newSpecs)
   127  				require.Subset(t, newSpecs, originals)
   128  			}
   129  		})
   130  	}
   131  
   132  }