github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/fs/operations/xtra_operations_test.go (about)

     1  // Extra operations tests (xtra_operations_test.go).
     2  //
     3  // This group contains tests, which involve streaming uploads.
     4  // Currently they are TestRcat and TestRcatSize, which directly
     5  // or implicitly invoke ioutil.NopCloser().
     6  // Indeterminate upload size triggers multi-upload in few backends.
     7  //
     8  // The S3 backend additionally triggers extra large upload buffers.
     9  // Namely, multiupload track in the Object.upload() method of S3
    10  // backend (rclone/backends/s3.go) selects PartSize about 512M,
    11  // upload.init() of AWS SDK (vendor/.../s3/s3manager/upload.go)
    12  // allocates upload.bufferPool of that size for each concurrent
    13  // upload goroutine. Given default concurrency of 4, this results
    14  // in 2G buffers persisting until the test executable ends.
    15  //
    16  // As the rclone test suite parallelizes test runs, this may
    17  // create memory pressure on a test box and trigger kernel swap,
    18  // which extremely slows down the test and makes probability
    19  // of memory contention between test processes even higher.
    20  //
    21  // Since name of this source file deliberately starts with `x`,
    22  // its tests will run lattermost isolating high memory at the
    23  // very end to somewhat reduce the contention probability.
    24  //
    25  package operations_test
    26  
    27  import (
    28  	"context"
    29  	"io/ioutil"
    30  	"strings"
    31  	"testing"
    32  
    33  	"github.com/rclone/rclone/fs"
    34  	"github.com/rclone/rclone/fs/operations"
    35  	"github.com/rclone/rclone/fstest"
    36  	"github.com/stretchr/testify/assert"
    37  	"github.com/stretchr/testify/require"
    38  )
    39  
    40  func TestRcat(t *testing.T) {
    41  	checkSumBefore := fs.Config.CheckSum
    42  	defer func() { fs.Config.CheckSum = checkSumBefore }()
    43  
    44  	check := func(withChecksum bool) {
    45  		fs.Config.CheckSum = withChecksum
    46  		prefix := "no_checksum_"
    47  		if withChecksum {
    48  			prefix = "with_checksum_"
    49  		}
    50  
    51  		r := fstest.NewRun(t)
    52  		defer r.Finalise()
    53  
    54  		if *fstest.SizeLimit > 0 && int64(fs.Config.StreamingUploadCutoff) > *fstest.SizeLimit {
    55  			savedCutoff := fs.Config.StreamingUploadCutoff
    56  			defer func() {
    57  				fs.Config.StreamingUploadCutoff = savedCutoff
    58  			}()
    59  			fs.Config.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit)
    60  			t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", fs.Config.StreamingUploadCutoff, savedCutoff)
    61  		}
    62  
    63  		fstest.CheckListing(t, r.Fremote, []fstest.Item{})
    64  
    65  		data1 := "this is some really nice test data"
    66  		path1 := prefix + "small_file_from_pipe"
    67  
    68  		data2 := string(make([]byte, fs.Config.StreamingUploadCutoff+1))
    69  		path2 := prefix + "big_file_from_pipe"
    70  
    71  		in := ioutil.NopCloser(strings.NewReader(data1))
    72  		_, err := operations.Rcat(context.Background(), r.Fremote, path1, in, t1)
    73  		require.NoError(t, err)
    74  
    75  		in = ioutil.NopCloser(strings.NewReader(data2))
    76  		_, err = operations.Rcat(context.Background(), r.Fremote, path2, in, t2)
    77  		require.NoError(t, err)
    78  
    79  		file1 := fstest.NewItem(path1, data1, t1)
    80  		file2 := fstest.NewItem(path2, data2, t2)
    81  		fstest.CheckItems(t, r.Fremote, file1, file2)
    82  	}
    83  
    84  	check(true)
    85  	check(false)
    86  }
    87  
    88  func TestRcatSize(t *testing.T) {
    89  	r := fstest.NewRun(t)
    90  	defer r.Finalise()
    91  
    92  	const body = "------------------------------------------------------------"
    93  	file1 := r.WriteFile("potato1", body, t1)
    94  	file2 := r.WriteFile("potato2", body, t2)
    95  	// Test with known length
    96  	bodyReader := ioutil.NopCloser(strings.NewReader(body))
    97  	obj, err := operations.RcatSize(context.Background(), r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
    98  	require.NoError(t, err)
    99  	assert.Equal(t, int64(len(body)), obj.Size())
   100  	assert.Equal(t, file1.Path, obj.Remote())
   101  
   102  	// Test with unknown length
   103  	bodyReader = ioutil.NopCloser(strings.NewReader(body)) // reset Reader
   104  	ioutil.NopCloser(strings.NewReader(body))
   105  	obj, err = operations.RcatSize(context.Background(), r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
   106  	require.NoError(t, err)
   107  	assert.Equal(t, int64(len(body)), obj.Size())
   108  	assert.Equal(t, file2.Path, obj.Remote())
   109  
   110  	// Check files exist
   111  	fstest.CheckItems(t, r.Fremote, file1, file2)
   112  }