github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/vfs/vfscache/downloaders/downloaders_test.go (about)

     1  package downloaders
     2  
     3  import (
     4  	"context"
     5  	"io"
     6  	"sync"
     7  	"testing"
     8  	"time"
     9  
    10  	_ "github.com/rclone/rclone/backend/local"
    11  	"github.com/rclone/rclone/fs/operations"
    12  	"github.com/rclone/rclone/fstest"
    13  	"github.com/rclone/rclone/lib/ranges"
    14  	"github.com/rclone/rclone/lib/readers"
    15  	"github.com/rclone/rclone/vfs/vfscommon"
    16  	"github.com/stretchr/testify/assert"
    17  	"github.com/stretchr/testify/require"
    18  )
    19  
    20  // TestMain drives the tests
    21  func TestMain(m *testing.M) {
    22  	fstest.TestMain(m)
    23  }
    24  
    25  type testItem struct {
    26  	mu   sync.Mutex
    27  	t    *testing.T
    28  	rs   ranges.Ranges
    29  	size int64
    30  }
    31  
    32  // HasRange returns true if the current ranges entirely include range
    33  func (item *testItem) HasRange(r ranges.Range) bool {
    34  	item.mu.Lock()
    35  	defer item.mu.Unlock()
    36  	return item.rs.Present(r)
    37  }
    38  
    39  // FindMissing adjusts r returning a new ranges.Range which only
    40  // contains the range which needs to be downloaded. This could be
    41  // empty - check with IsEmpty. It also adjust this to make sure it is
    42  // not larger than the file.
    43  func (item *testItem) FindMissing(r ranges.Range) (outr ranges.Range) {
    44  	item.mu.Lock()
    45  	defer item.mu.Unlock()
    46  	outr = item.rs.FindMissing(r)
    47  	// Clip returned block to size of file
    48  	outr.Clip(item.size)
    49  	return outr
    50  }
    51  
    52  // WriteAtNoOverwrite writes b to the file, but will not overwrite
    53  // already present ranges.
    54  //
    55  // This is used by the downloader to write bytes to the file.
    56  //
    57  // It returns n the total bytes processed and skipped the number of
    58  // bytes which were processed but not actually written to the file.
    59  func (item *testItem) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, err error) {
    60  	item.mu.Lock()
    61  	defer item.mu.Unlock()
    62  	item.rs.Insert(ranges.Range{Pos: off, Size: int64(len(b))})
    63  
    64  	// Check contents is correct
    65  	in := readers.NewPatternReader(item.size)
    66  	checkBuf := make([]byte, len(b))
    67  	_, err = in.Seek(off, io.SeekStart)
    68  	require.NoError(item.t, err)
    69  	n, _ = in.Read(checkBuf)
    70  	require.Equal(item.t, len(b), n)
    71  	assert.Equal(item.t, checkBuf, b)
    72  
    73  	return n, 0, nil
    74  }
    75  
    76  func TestDownloaders(t *testing.T) {
    77  	r := fstest.NewRun(t)
    78  
    79  	var (
    80  		ctx    = context.Background()
    81  		remote = "potato.txt"
    82  		size   = int64(50*1024*1024 - 1234)
    83  	)
    84  
    85  	// Write the test file
    86  	in := io.NopCloser(readers.NewPatternReader(size))
    87  	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now(), nil)
    88  	require.NoError(t, err)
    89  	assert.Equal(t, size, src.Size())
    90  
    91  	newTest := func() (*testItem, *Downloaders) {
    92  		item := &testItem{
    93  			t:    t,
    94  			size: size,
    95  		}
    96  		opt := vfscommon.DefaultOpt
    97  		dls := New(item, &opt, remote, src)
    98  		return item, dls
    99  	}
   100  	cancel := func(dls *Downloaders) {
   101  		assert.NoError(t, dls.Close(nil))
   102  	}
   103  
   104  	t.Run("Download", func(t *testing.T) {
   105  		item, dls := newTest()
   106  		defer cancel(dls)
   107  
   108  		for _, r := range []ranges.Range{
   109  			{Pos: 100, Size: 250},
   110  			{Pos: 500, Size: 250},
   111  			{Pos: 25000000, Size: 250},
   112  		} {
   113  			err := dls.Download(r)
   114  			require.NoError(t, err)
   115  			assert.True(t, item.HasRange(r))
   116  		}
   117  	})
   118  
   119  	t.Run("EnsureDownloader", func(t *testing.T) {
   120  		item, dls := newTest()
   121  		defer cancel(dls)
   122  		r := ranges.Range{Pos: 40 * 1024 * 1024, Size: 250}
   123  		err := dls.EnsureDownloader(r)
   124  		require.NoError(t, err)
   125  		// FIXME racy test
   126  		assert.False(t, item.HasRange(r))
   127  		time.Sleep(time.Second)
   128  		assert.True(t, item.HasRange(r))
   129  	})
   130  }