github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fs/operations/multithread_test.go (about)

     1  package operations
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"testing"
     7  
     8  	"github.com/rclone/rclone/fs/accounting"
     9  	"github.com/rclone/rclone/fstest/mockfs"
    10  	"github.com/rclone/rclone/fstest/mockobject"
    11  	"github.com/rclone/rclone/lib/random"
    12  
    13  	"github.com/rclone/rclone/fs"
    14  	"github.com/rclone/rclone/fstest"
    15  	"github.com/stretchr/testify/assert"
    16  	"github.com/stretchr/testify/require"
    17  )
    18  
    19  func TestDoMultiThreadCopy(t *testing.T) {
    20  	f := mockfs.NewFs("potato", "")
    21  	src := mockobject.New("file.txt").WithContent([]byte(random.String(100)), mockobject.SeekModeNone)
    22  	srcFs := mockfs.NewFs("sausage", "")
    23  	src.SetFs(srcFs)
    24  
    25  	oldStreams := fs.Config.MultiThreadStreams
    26  	oldCutoff := fs.Config.MultiThreadCutoff
    27  	oldIsSet := fs.Config.MultiThreadSet
    28  	defer func() {
    29  		fs.Config.MultiThreadStreams = oldStreams
    30  		fs.Config.MultiThreadCutoff = oldCutoff
    31  		fs.Config.MultiThreadSet = oldIsSet
    32  	}()
    33  
    34  	fs.Config.MultiThreadStreams, fs.Config.MultiThreadCutoff = 4, 50
    35  	fs.Config.MultiThreadSet = false
    36  
    37  	nullWriterAt := func(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
    38  		panic("don't call me")
    39  	}
    40  	f.Features().OpenWriterAt = nullWriterAt
    41  
    42  	assert.True(t, doMultiThreadCopy(f, src))
    43  
    44  	fs.Config.MultiThreadStreams = 0
    45  	assert.False(t, doMultiThreadCopy(f, src))
    46  	fs.Config.MultiThreadStreams = 1
    47  	assert.False(t, doMultiThreadCopy(f, src))
    48  	fs.Config.MultiThreadStreams = 2
    49  	assert.True(t, doMultiThreadCopy(f, src))
    50  
    51  	fs.Config.MultiThreadCutoff = 200
    52  	assert.False(t, doMultiThreadCopy(f, src))
    53  	fs.Config.MultiThreadCutoff = 101
    54  	assert.False(t, doMultiThreadCopy(f, src))
    55  	fs.Config.MultiThreadCutoff = 100
    56  	assert.True(t, doMultiThreadCopy(f, src))
    57  
    58  	f.Features().OpenWriterAt = nil
    59  	assert.False(t, doMultiThreadCopy(f, src))
    60  	f.Features().OpenWriterAt = nullWriterAt
    61  	assert.True(t, doMultiThreadCopy(f, src))
    62  
    63  	f.Features().IsLocal = true
    64  	srcFs.Features().IsLocal = true
    65  	assert.False(t, doMultiThreadCopy(f, src))
    66  	fs.Config.MultiThreadSet = true
    67  	assert.True(t, doMultiThreadCopy(f, src))
    68  	fs.Config.MultiThreadSet = false
    69  	assert.False(t, doMultiThreadCopy(f, src))
    70  	srcFs.Features().IsLocal = false
    71  	assert.True(t, doMultiThreadCopy(f, src))
    72  	srcFs.Features().IsLocal = true
    73  	assert.False(t, doMultiThreadCopy(f, src))
    74  	f.Features().IsLocal = false
    75  	assert.True(t, doMultiThreadCopy(f, src))
    76  	srcFs.Features().IsLocal = false
    77  	assert.True(t, doMultiThreadCopy(f, src))
    78  }
    79  
    80  func TestMultithreadCalculateChunks(t *testing.T) {
    81  	for _, test := range []struct {
    82  		size         int64
    83  		streams      int
    84  		wantPartSize int64
    85  		wantStreams  int
    86  	}{
    87  		{size: 1, streams: 10, wantPartSize: multithreadChunkSize, wantStreams: 1},
    88  		{size: 1 << 20, streams: 1, wantPartSize: 1 << 20, wantStreams: 1},
    89  		{size: 1 << 20, streams: 2, wantPartSize: 1 << 19, wantStreams: 2},
    90  		{size: (1 << 20) + 1, streams: 2, wantPartSize: (1 << 19) + multithreadChunkSize, wantStreams: 2},
    91  		{size: (1 << 20) - 1, streams: 2, wantPartSize: (1 << 19), wantStreams: 2},
    92  	} {
    93  		t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) {
    94  			mc := &multiThreadCopyState{
    95  				size:    test.size,
    96  				streams: test.streams,
    97  			}
    98  			mc.calculateChunks()
    99  			assert.Equal(t, test.wantPartSize, mc.partSize)
   100  			assert.Equal(t, test.wantStreams, mc.streams)
   101  		})
   102  	}
   103  }
   104  
   105  func TestMultithreadCopy(t *testing.T) {
   106  	r := fstest.NewRun(t)
   107  	defer r.Finalise()
   108  
   109  	for _, test := range []struct {
   110  		size    int
   111  		streams int
   112  	}{
   113  		{size: multithreadChunkSize*2 - 1, streams: 2},
   114  		{size: multithreadChunkSize * 2, streams: 2},
   115  		{size: multithreadChunkSize*2 + 1, streams: 2},
   116  	} {
   117  		t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) {
   118  			if *fstest.SizeLimit > 0 && int64(test.size) > *fstest.SizeLimit {
   119  				t.Skipf("exceeded file size limit %d > %d", test.size, *fstest.SizeLimit)
   120  			}
   121  			var err error
   122  			contents := random.String(test.size)
   123  			t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
   124  			file1 := r.WriteObject(context.Background(), "file1", contents, t1)
   125  			fstest.CheckItems(t, r.Fremote, file1)
   126  			fstest.CheckItems(t, r.Flocal)
   127  
   128  			src, err := r.Fremote.NewObject(context.Background(), "file1")
   129  			require.NoError(t, err)
   130  			accounting.GlobalStats().ResetCounters()
   131  			tr := accounting.GlobalStats().NewTransfer(src)
   132  
   133  			defer func() {
   134  				tr.Done(err)
   135  			}()
   136  			dst, err := multiThreadCopy(context.Background(), r.Flocal, "file1", src, 2, tr)
   137  			require.NoError(t, err)
   138  			assert.Equal(t, src.Size(), dst.Size())
   139  			assert.Equal(t, "file1", dst.Remote())
   140  
   141  			fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1}, nil, fs.GetModifyWindow(r.Flocal, r.Fremote))
   142  			require.NoError(t, dst.Remove(context.Background()))
   143  		})
   144  	}
   145  
   146  }