github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/fs/operations/dedupe_test.go (about)

     1  package operations_test
     2  
     3  import (
     4  	"context"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/ncw/rclone/fs"
     9  	"github.com/ncw/rclone/fs/hash"
    10  	"github.com/ncw/rclone/fs/operations"
    11  	"github.com/ncw/rclone/fs/walk"
    12  	"github.com/ncw/rclone/fstest"
    13  	"github.com/stretchr/testify/assert"
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  func skipIfCantDedupe(t *testing.T, f fs.Fs) {
    18  	if !f.Features().DuplicateFiles {
    19  		t.Skip("Can't test deduplicate - no duplicate files possible")
    20  	}
    21  	if f.Features().PutUnchecked == nil {
    22  		t.Skip("Can't test deduplicate - no PutUnchecked")
    23  	}
    24  	if f.Features().MergeDirs == nil {
    25  		t.Skip("Can't test deduplicate - no MergeDirs")
    26  	}
    27  }
    28  
    29  func skipIfNoHash(t *testing.T, f fs.Fs) {
    30  	if f.Hashes().GetOne() == hash.None {
    31  		t.Skip("Can't run this test without a hash")
    32  	}
    33  }
    34  
    35  func TestDeduplicateInteractive(t *testing.T) {
    36  	r := fstest.NewRun(t)
    37  	defer r.Finalise()
    38  	skipIfCantDedupe(t, r.Fremote)
    39  	skipIfNoHash(t, r.Fremote)
    40  
    41  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    42  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    43  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    44  	r.CheckWithDuplicates(t, file1, file2, file3)
    45  
    46  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive)
    47  	require.NoError(t, err)
    48  
    49  	fstest.CheckItems(t, r.Fremote, file1)
    50  }
    51  
    52  func TestDeduplicateSkip(t *testing.T) {
    53  	r := fstest.NewRun(t)
    54  	defer r.Finalise()
    55  	skipIfCantDedupe(t, r.Fremote)
    56  	haveHash := r.Fremote.Hashes().GetOne() != hash.None
    57  
    58  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    59  	files := []fstest.Item{file1}
    60  	if haveHash {
    61  		file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    62  		files = append(files, file2)
    63  	}
    64  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
    65  	files = append(files, file3)
    66  	r.CheckWithDuplicates(t, files...)
    67  
    68  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip)
    69  	require.NoError(t, err)
    70  
    71  	r.CheckWithDuplicates(t, file1, file3)
    72  }
    73  
    74  func TestDeduplicateFirst(t *testing.T) {
    75  	r := fstest.NewRun(t)
    76  	defer r.Finalise()
    77  	skipIfCantDedupe(t, r.Fremote)
    78  
    79  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
    80  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1)
    81  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1)
    82  	r.CheckWithDuplicates(t, file1, file2, file3)
    83  
    84  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst)
    85  	require.NoError(t, err)
    86  
    87  	// list until we get one object
    88  	var objects, size int64
    89  	for try := 1; try <= *fstest.ListRetries; try++ {
    90  		objects, size, err = operations.Count(context.Background(), r.Fremote)
    91  		require.NoError(t, err)
    92  		if objects == 1 {
    93  			break
    94  		}
    95  		time.Sleep(time.Second)
    96  	}
    97  	assert.Equal(t, int64(1), objects)
    98  	if size != file1.Size && size != file2.Size && size != file3.Size {
    99  		t.Errorf("Size not one of the object sizes %d", size)
   100  	}
   101  }
   102  
   103  func TestDeduplicateNewest(t *testing.T) {
   104  	r := fstest.NewRun(t)
   105  	defer r.Finalise()
   106  	skipIfCantDedupe(t, r.Fremote)
   107  
   108  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   109  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   110  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   111  	r.CheckWithDuplicates(t, file1, file2, file3)
   112  
   113  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest)
   114  	require.NoError(t, err)
   115  
   116  	fstest.CheckItems(t, r.Fremote, file3)
   117  }
   118  
   119  func TestDeduplicateOldest(t *testing.T) {
   120  	r := fstest.NewRun(t)
   121  	defer r.Finalise()
   122  	skipIfCantDedupe(t, r.Fremote)
   123  
   124  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   125  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   126  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   127  	r.CheckWithDuplicates(t, file1, file2, file3)
   128  
   129  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest)
   130  	require.NoError(t, err)
   131  
   132  	fstest.CheckItems(t, r.Fremote, file1)
   133  }
   134  
   135  func TestDeduplicateLargest(t *testing.T) {
   136  	r := fstest.NewRun(t)
   137  	defer r.Finalise()
   138  	skipIfCantDedupe(t, r.Fremote)
   139  
   140  	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
   141  	file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
   142  	file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
   143  	r.CheckWithDuplicates(t, file1, file2, file3)
   144  
   145  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest)
   146  	require.NoError(t, err)
   147  
   148  	fstest.CheckItems(t, r.Fremote, file3)
   149  }
   150  
   151  func TestDeduplicateRename(t *testing.T) {
   152  	r := fstest.NewRun(t)
   153  	defer r.Finalise()
   154  	skipIfCantDedupe(t, r.Fremote)
   155  
   156  	file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
   157  	file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2)
   158  	file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3)
   159  	file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1)
   160  	r.CheckWithDuplicates(t, file1, file2, file3, file4)
   161  
   162  	err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename)
   163  	require.NoError(t, err)
   164  
   165  	require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
   166  		entries.ForObject(func(o fs.Object) {
   167  			remote := o.Remote()
   168  			if remote != "one-1.txt" &&
   169  				remote != "one-2.txt" &&
   170  				remote != "one-3.txt" &&
   171  				remote != "one-4.txt" {
   172  				t.Errorf("Bad file name after rename %q", remote)
   173  			}
   174  			size := o.Size()
   175  			if size != file1.Size &&
   176  				size != file2.Size &&
   177  				size != file3.Size &&
   178  				size != file4.Size {
   179  				t.Errorf("Size not one of the object sizes %d", size)
   180  			}
   181  			if remote == "one-1.txt" && size != file4.Size {
   182  				t.Errorf("Existing non-duplicate file modified %q", remote)
   183  			}
   184  		})
   185  		return nil
   186  	}))
   187  }
   188  
   189  // This should really be a unit test, but the test framework there
   190  // doesn't have enough tools to make it easy
   191  func TestMergeDirs(t *testing.T) {
   192  	r := fstest.NewRun(t)
   193  	defer r.Finalise()
   194  
   195  	mergeDirs := r.Fremote.Features().MergeDirs
   196  	if mergeDirs == nil {
   197  		t.Skip("Can't merge directories")
   198  	}
   199  
   200  	file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1)
   201  	file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2)
   202  	file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3)
   203  
   204  	objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1)
   205  	require.NoError(t, err)
   206  	assert.Equal(t, 3, len(dirs))
   207  	assert.Equal(t, 0, len(objs))
   208  
   209  	err = mergeDirs(context.Background(), dirs)
   210  	require.NoError(t, err)
   211  
   212  	file2.Path = "dupe1/two.txt"
   213  	file3.Path = "dupe1/three.txt"
   214  	fstest.CheckItems(t, r.Fremote, file1, file2, file3)
   215  
   216  	objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1)
   217  	require.NoError(t, err)
   218  	assert.Equal(t, 1, len(dirs))
   219  	assert.Equal(t, 0, len(objs))
   220  	assert.Equal(t, "dupe1", dirs[0].Remote())
   221  }