github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fs/operations/dedupe_test.go (about)

     1  package operations_test
     2  
     3  import (
     4  	"context"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/rclone/rclone/fs"
     9  	"github.com/rclone/rclone/fs/hash"
    10  	"github.com/rclone/rclone/fs/operations"
    11  	"github.com/rclone/rclone/fs/walk"
    12  	"github.com/rclone/rclone/fstest"
    13  	"github.com/spf13/pflag"
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  // Check flag satisfies the interface
    19  var _ pflag.Value = (*operations.DeduplicateMode)(nil)
    20  
    21  func skipIfCantDedupe(t *testing.T, f fs.Fs) {
    22  	if !f.Features().DuplicateFiles {
    23  		t.Skip("Can't test deduplicate - no duplicate files possible")
    24  	}
    25  	if f.Features().PutUnchecked == nil {
    26  		t.Skip("Can't test deduplicate - no PutUnchecked")
    27  	}
    28  	if f.Features().MergeDirs == nil {
    29  		t.Skip("Can't test deduplicate - no MergeDirs")
    30  	}
    31  }
    32  
    33  func skipIfNoHash(t *testing.T, f fs.Fs) {
    34  	if f.Hashes().GetOne() == hash.None {
    35  		t.Skip("Can't run this test without a hash")
    36  	}
    37  }
    38  
    39  func TestDeduplicateInteractive(t *testing.T) {
    40  	r := fstest.NewRun(t)
    41  	defer r.Finalise()
    42  	skipIfCantDedupe(t, r.Fremote)
    43  	skipIfNoHash(t, r.Fremote)
    44  
    45  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    46  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    47  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    48  	r.CheckWithDuplicates(t, file1, file2, file3)
    49  
    50  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive)
    51  	require.NoError(t, err)
    52  
    53  	fstest.CheckItems(t, r.Fremote, file1)
    54  }
    55  
    56  func TestDeduplicateSkip(t *testing.T) {
    57  	r := fstest.NewRun(t)
    58  	defer r.Finalise()
    59  	skipIfCantDedupe(t, r.Fremote)
    60  	haveHash := r.Fremote.Hashes().GetOne() != hash.None
    61  
    62  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    63  	files := []fstest.Item{file1}
    64  	if haveHash {
    65  		file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    66  		files = append(files, file2)
    67  	}
    68  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
    69  	files = append(files, file3)
    70  	r.CheckWithDuplicates(t, files...)
    71  
    72  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip)
    73  	require.NoError(t, err)
    74  
    75  	r.CheckWithDuplicates(t, file1, file3)
    76  }
    77  
    78  func TestDeduplicateFirst(t *testing.T) {
    79  	r := fstest.NewRun(t)
    80  	defer r.Finalise()
    81  	skipIfCantDedupe(t, r.Fremote)
    82  
    83  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    84  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1)
    85  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1)
    86  	r.CheckWithDuplicates(t, file1, file2, file3)
    87  
    88  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst)
    89  	require.NoError(t, err)
    90  
    91  	// list until we get one object
    92  	var objects, size int64
    93  	for try := 1; try <= *fstest.ListRetries; try++ {
    94  		objects, size, err = operations.Count(context.Background(), r.Fremote)
    95  		require.NoError(t, err)
    96  		if objects == 1 {
    97  			break
    98  		}
    99  		time.Sleep(time.Second)
   100  	}
   101  	assert.Equal(t, int64(1), objects)
   102  	if size != file1.Size && size != file2.Size && size != file3.Size {
   103  		t.Errorf("Size not one of the object sizes %d", size)
   104  	}
   105  }
   106  
   107  func TestDeduplicateNewest(t *testing.T) {
   108  	r := fstest.NewRun(t)
   109  	defer r.Finalise()
   110  	skipIfCantDedupe(t, r.Fremote)
   111  
   112  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   113  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   114  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   115  	r.CheckWithDuplicates(t, file1, file2, file3)
   116  
   117  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest)
   118  	require.NoError(t, err)
   119  
   120  	fstest.CheckItems(t, r.Fremote, file3)
   121  }
   122  
   123  func TestDeduplicateOldest(t *testing.T) {
   124  	r := fstest.NewRun(t)
   125  	defer r.Finalise()
   126  	skipIfCantDedupe(t, r.Fremote)
   127  
   128  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   129  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   130  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   131  	r.CheckWithDuplicates(t, file1, file2, file3)
   132  
   133  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest)
   134  	require.NoError(t, err)
   135  
   136  	fstest.CheckItems(t, r.Fremote, file1)
   137  }
   138  
   139  func TestDeduplicateLargest(t *testing.T) {
   140  	r := fstest.NewRun(t)
   141  	defer r.Finalise()
   142  	skipIfCantDedupe(t, r.Fremote)
   143  
   144  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   145  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   146  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   147  	r.CheckWithDuplicates(t, file1, file2, file3)
   148  
   149  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest)
   150  	require.NoError(t, err)
   151  
   152  	fstest.CheckItems(t, r.Fremote, file3)
   153  }
   154  
   155  func TestDeduplicateSmallest(t *testing.T) {
   156  	r := fstest.NewRun(t)
   157  	defer r.Finalise()
   158  	skipIfCantDedupe(t, r.Fremote)
   159  
   160  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   161  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   162  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   163  	r.CheckWithDuplicates(t, file1, file2, file3)
   164  
   165  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSmallest)
   166  	require.NoError(t, err)
   167  
   168  	fstest.CheckItems(t, r.Fremote, file1)
   169  }
   170  
   171  func TestDeduplicateRename(t *testing.T) {
   172  	r := fstest.NewRun(t)
   173  	defer r.Finalise()
   174  	skipIfCantDedupe(t, r.Fremote)
   175  
   176  	file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
   177  	file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2)
   178  	file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3)
   179  	file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1)
   180  	r.CheckWithDuplicates(t, file1, file2, file3, file4)
   181  
   182  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename)
   183  	require.NoError(t, err)
   184  
   185  	require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
   186  		entries.ForObject(func(o fs.Object) {
   187  			remote := o.Remote()
   188  			if remote != "one-1.txt" &&
   189  				remote != "one-2.txt" &&
   190  				remote != "one-3.txt" &&
   191  				remote != "one-4.txt" {
   192  				t.Errorf("Bad file name after rename %q", remote)
   193  			}
   194  			size := o.Size()
   195  			if size != file1.Size &&
   196  				size != file2.Size &&
   197  				size != file3.Size &&
   198  				size != file4.Size {
   199  				t.Errorf("Size not one of the object sizes %d", size)
   200  			}
   201  			if remote == "one-1.txt" && size != file4.Size {
   202  				t.Errorf("Existing non-duplicate file modified %q", remote)
   203  			}
   204  		})
   205  		return nil
   206  	}))
   207  }
   208  
   209  // This should really be a unit test, but the test framework there
   210  // doesn't have enough tools to make it easy
   211  func TestMergeDirs(t *testing.T) {
   212  	r := fstest.NewRun(t)
   213  	defer r.Finalise()
   214  
   215  	mergeDirs := r.Fremote.Features().MergeDirs
   216  	if mergeDirs == nil {
   217  		t.Skip("Can't merge directories")
   218  	}
   219  
   220  	file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1)
   221  	file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2)
   222  	file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3)
   223  
   224  	objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1)
   225  	require.NoError(t, err)
   226  	assert.Equal(t, 3, len(dirs))
   227  	assert.Equal(t, 0, len(objs))
   228  
   229  	err = mergeDirs(context.Background(), dirs)
   230  	require.NoError(t, err)
   231  
   232  	file2.Path = "dupe1/two.txt"
   233  	file3.Path = "dupe1/three.txt"
   234  	fstest.CheckItems(t, r.Fremote, file1, file2, file3)
   235  
   236  	objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1)
   237  	require.NoError(t, err)
   238  	assert.Equal(t, 1, len(dirs))
   239  	assert.Equal(t, 0, len(objs))
   240  	assert.Equal(t, "dupe1", dirs[0].Remote())
   241  }