github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/b2/b2_internal_test.go (about)

     1  package b2
     2  
     3  import (
     4  	"context"
     5  	"crypto/sha1"
     6  	"fmt"
     7  	"path"
     8  	"strings"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/rclone/rclone/backend/b2/api"
    13  	"github.com/rclone/rclone/fs"
    14  	"github.com/rclone/rclone/fs/cache"
    15  	"github.com/rclone/rclone/fs/hash"
    16  	"github.com/rclone/rclone/fstest"
    17  	"github.com/rclone/rclone/fstest/fstests"
    18  	"github.com/rclone/rclone/lib/bucket"
    19  	"github.com/rclone/rclone/lib/random"
    20  	"github.com/rclone/rclone/lib/version"
    21  	"github.com/stretchr/testify/assert"
    22  	"github.com/stretchr/testify/require"
    23  )
    24  
    25  // Test b2 string encoding
    26  // https://www.backblaze.com/b2/docs/string_encoding.html
    27  
    28  var encodeTest = []struct {
    29  	fullyEncoded     string
    30  	minimallyEncoded string
    31  	plainText        string
    32  }{
    33  	{fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "},
    34  	{fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"},
    35  	{fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""},
    36  	{fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"},
    37  	{fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"},
    38  	{fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"},
    39  	{fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"},
    40  	{fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"},
    41  	{fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("},
    42  	{fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"},
    43  	{fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"},
    44  	{fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"},
    45  	{fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","},
    46  	{fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"},
    47  	{fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."},
    48  	{fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"},
    49  	{fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"},
    50  	{fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"},
    51  	{fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"},
    52  	{fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"},
    53  	{fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"},
    54  	{fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"},
    55  	{fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"},
    56  	{fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"},
    57  	{fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"},
    58  	{fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"},
    59  	{fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"},
    60  	{fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"},
    61  	{fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"},
    62  	{fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="},
    63  	{fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"},
    64  	{fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"},
    65  	{fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"},
    66  	{fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"},
    67  	{fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"},
    68  	{fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"},
    69  	{fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"},
    70  	{fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"},
    71  	{fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"},
    72  	{fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"},
    73  	{fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"},
    74  	{fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"},
    75  	{fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"},
    76  	{fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"},
    77  	{fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"},
    78  	{fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"},
    79  	{fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"},
    80  	{fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"},
    81  	{fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"},
    82  	{fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"},
    83  	{fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"},
    84  	{fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"},
    85  	{fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"},
    86  	{fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"},
    87  	{fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"},
    88  	{fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"},
    89  	{fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"},
    90  	{fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"},
    91  	{fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"},
    92  	{fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["},
    93  	{fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"},
    94  	{fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"},
    95  	{fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"},
    96  	{fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"},
    97  	{fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"},
    98  	{fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"},
    99  	{fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"},
   100  	{fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"},
   101  	{fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"},
   102  	{fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"},
   103  	{fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"},
   104  	{fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"},
   105  	{fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"},
   106  	{fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"},
   107  	{fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"},
   108  	{fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"},
   109  	{fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"},
   110  	{fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"},
   111  	{fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"},
   112  	{fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"},
   113  	{fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"},
   114  	{fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"},
   115  	{fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"},
   116  	{fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"},
   117  	{fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"},
   118  	{fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"},
   119  	{fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"},
   120  	{fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"},
   121  	{fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"},
   122  	{fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"},
   123  	{fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"},
   124  	{fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"},
   125  	{fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"},
   126  	{fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"},
   127  	{fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"},
   128  	{fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"},
   129  	{fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"},
   130  	{fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"},
   131  }
   132  
   133  func TestUrlEncode(t *testing.T) {
   134  	for _, test := range encodeTest {
   135  		got := urlEncode(test.plainText)
   136  		if got != test.minimallyEncoded && got != test.fullyEncoded {
   137  			t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded)
   138  		}
   139  	}
   140  }
   141  
   142  func TestTimeString(t *testing.T) {
   143  	for _, test := range []struct {
   144  		in   time.Time
   145  		want string
   146  	}{
   147  		{fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"},
   148  		{fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"},
   149  		{fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"},
   150  	} {
   151  		got := timeString(test.in)
   152  		if test.want != got {
   153  			t.Logf("%v: want %v got %v", test.in, test.want, got)
   154  		}
   155  	}
   156  
   157  }
   158  
   159  func TestParseTimeString(t *testing.T) {
   160  	for _, test := range []struct {
   161  		in        string
   162  		want      time.Time
   163  		wantError string
   164  	}{
   165  		{"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""},
   166  		{"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""},
   167  		{"", time.Time{}, ""},
   168  		{"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`},
   169  	} {
   170  		o := Object{}
   171  		err := o.parseTimeString(test.in)
   172  		got := o.modTime
   173  		var gotError string
   174  		if err != nil {
   175  			gotError = err.Error()
   176  		}
   177  		if test.want != got {
   178  			t.Logf("%v: want %v got %v", test.in, test.want, got)
   179  		}
   180  		if test.wantError != gotError {
   181  			t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError)
   182  		}
   183  	}
   184  
   185  }
   186  
   187  // This is adapted from the s3 equivalent.
   188  func (f *Fs) InternalTestMetadata(t *testing.T) {
   189  	ctx := context.Background()
   190  	original := random.String(1000)
   191  	contents := fstest.Gz(t, original)
   192  	mimeType := "text/html"
   193  
   194  	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
   195  	btime := time.Now()
   196  	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, mimeType, nil)
   197  	defer func() {
   198  		assert.NoError(t, obj.Remove(ctx))
   199  	}()
   200  	o := obj.(*Object)
   201  	gotMetadata, err := o.getMetaData(ctx)
   202  	require.NoError(t, err)
   203  
   204  	// We currently have a limited amount of metadata to test with B2
   205  	assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
   206  
   207  	// Modification time from the x-bz-info-src_last_modified_millis header
   208  	var mtime api.Timestamp
   209  	err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
   210  	if err != nil {
   211  		fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
   212  	}
   213  	assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
   214  
   215  	// Upload time
   216  	gotBtime := time.Time(gotMetadata.UploadTimestamp)
   217  	dt := gotBtime.Sub(btime)
   218  	assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
   219  
   220  	t.Run("GzipEncoding", func(t *testing.T) {
   221  		// Test that the gzipped file we uploaded can be
   222  		// downloaded
   223  		checkDownload := func(wantContents string, wantSize int64, wantHash string) {
   224  			gotContents := fstests.ReadObject(ctx, t, o, -1)
   225  			assert.Equal(t, wantContents, gotContents)
   226  			assert.Equal(t, wantSize, o.Size())
   227  			gotHash, err := o.Hash(ctx, hash.SHA1)
   228  			require.NoError(t, err)
   229  			assert.Equal(t, wantHash, gotHash)
   230  		}
   231  
   232  		t.Run("NoDecompress", func(t *testing.T) {
   233  			checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
   234  		})
   235  	})
   236  }
   237  
   238  func sha1Sum(t *testing.T, s string) string {
   239  	hash := sha1.Sum([]byte(s))
   240  	return fmt.Sprintf("%x", hash)
   241  }
   242  
   243  // This is adapted from the s3 equivalent.
   244  func (f *Fs) InternalTestVersions(t *testing.T) {
   245  	ctx := context.Background()
   246  
   247  	// Small pause to make the LastModified different since AWS
   248  	// only seems to track them to 1 second granularity
   249  	time.Sleep(2 * time.Second)
   250  
   251  	// Create an object
   252  	const dirName = "versions"
   253  	const fileName = dirName + "/" + "test-versions.txt"
   254  	contents := random.String(100)
   255  	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
   256  	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
   257  	defer func() {
   258  		assert.NoError(t, obj.Remove(ctx))
   259  	}()
   260  	objMetadata, err := obj.(*Object).getMetaData(ctx)
   261  	require.NoError(t, err)
   262  
   263  	// Small pause
   264  	time.Sleep(2 * time.Second)
   265  
   266  	// Remove it
   267  	assert.NoError(t, obj.Remove(ctx))
   268  
   269  	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
   270  	time.Sleep(2 * time.Second)
   271  
   272  	// And create it with different size and contents
   273  	newContents := random.String(101)
   274  	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
   275  	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
   276  	newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
   277  	require.NoError(t, err)
   278  
   279  	t.Run("Versions", func(t *testing.T) {
   280  		// Set --b2-versions for this test
   281  		f.opt.Versions = true
   282  		defer func() {
   283  			f.opt.Versions = false
   284  		}()
   285  
   286  		// Read the contents
   287  		entries, err := f.List(ctx, dirName)
   288  		require.NoError(t, err)
   289  		tests := 0
   290  		var fileNameVersion string
   291  		for _, entry := range entries {
   292  			t.Log(entry)
   293  			remote := entry.Remote()
   294  			if remote == fileName {
   295  				t.Run("ReadCurrent", func(t *testing.T) {
   296  					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
   297  				})
   298  				tests++
   299  			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
   300  				t.Run("ReadVersion", func(t *testing.T) {
   301  					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
   302  				})
   303  				assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
   304  				fileNameVersion = remote
   305  				tests++
   306  			}
   307  		}
   308  		assert.Equal(t, 2, tests, "object missing from listing")
   309  
   310  		// Check we can read the object with a version suffix
   311  		t.Run("NewObject", func(t *testing.T) {
   312  			o, err := f.NewObject(ctx, fileNameVersion)
   313  			require.NoError(t, err)
   314  			require.NotNil(t, o)
   315  			assert.Equal(t, int64(100), o.Size(), o.Remote())
   316  		})
   317  
   318  		// Check we can make a NewFs from that object with a version suffix
   319  		t.Run("NewFs", func(t *testing.T) {
   320  			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
   321  			// Make sure --b2-versions is set in the config of the new remote
   322  			fs.Debugf(nil, "oldPath = %q", newPath)
   323  			lastColon := strings.LastIndex(newPath, ":")
   324  			require.True(t, lastColon >= 0)
   325  			newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
   326  			fs.Debugf(nil, "newPath = %q", newPath)
   327  			fNew, err := cache.Get(ctx, newPath)
   328  			// This should return pointing to a file
   329  			require.Equal(t, fs.ErrorIsFile, err)
   330  			require.NotNil(t, fNew)
   331  			// With the directory above
   332  			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
   333  		})
   334  	})
   335  
   336  	t.Run("VersionAt", func(t *testing.T) {
   337  		// We set --b2-version-at for this test so make sure we reset it at the end
   338  		defer func() {
   339  			f.opt.VersionAt = fs.Time{}
   340  		}()
   341  
   342  		var (
   343  			firstObjectTime  = time.Time(objMetadata.UploadTimestamp)
   344  			secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
   345  		)
   346  
   347  		for _, test := range []struct {
   348  			what     string
   349  			at       time.Time
   350  			want     []fstest.Item
   351  			wantErr  error
   352  			wantSize int64
   353  		}{
   354  			{
   355  				what:    "Before",
   356  				at:      firstObjectTime.Add(-time.Second),
   357  				want:    fstests.InternalTestFiles,
   358  				wantErr: fs.ErrorObjectNotFound,
   359  			},
   360  			{
   361  				what:     "AfterOne",
   362  				at:       firstObjectTime.Add(time.Second),
   363  				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
   364  				wantSize: 100,
   365  			},
   366  			{
   367  				what:    "AfterDelete",
   368  				at:      secondObjectTime.Add(-time.Second),
   369  				want:    fstests.InternalTestFiles,
   370  				wantErr: fs.ErrorObjectNotFound,
   371  			},
   372  			{
   373  				what:     "AfterTwo",
   374  				at:       secondObjectTime.Add(time.Second),
   375  				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
   376  				wantSize: 101,
   377  			},
   378  		} {
   379  			t.Run(test.what, func(t *testing.T) {
   380  				f.opt.VersionAt = fs.Time(test.at)
   381  				t.Run("List", func(t *testing.T) {
   382  					fstest.CheckListing(t, f, test.want)
   383  				})
   384  				// b2 NewObject doesn't work with VersionAt
   385  				//t.Run("NewObject", func(t *testing.T) {
   386  				//	gotObj, gotErr := f.NewObject(ctx, fileName)
   387  				//	assert.Equal(t, test.wantErr, gotErr)
   388  				//	if gotErr == nil {
   389  				//		assert.Equal(t, test.wantSize, gotObj.Size())
   390  				//	}
   391  				//})
   392  			})
   393  		}
   394  	})
   395  
   396  	t.Run("Cleanup", func(t *testing.T) {
   397  		require.NoError(t, f.cleanUp(ctx, true, false, 0))
   398  		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
   399  		fstest.CheckListing(t, f, items)
   400  		// Set --b2-versions for this test
   401  		f.opt.Versions = true
   402  		defer func() {
   403  			f.opt.Versions = false
   404  		}()
   405  		fstest.CheckListing(t, f, items)
   406  	})
   407  
   408  	// Purge gets tested later
   409  }
   410  
   411  // -run TestIntegration/FsMkdir/FsPutFiles/Internal
   412  func (f *Fs) InternalTest(t *testing.T) {
   413  	t.Run("Metadata", f.InternalTestMetadata)
   414  	t.Run("Versions", f.InternalTestVersions)
   415  }
   416  
   417  var _ fstests.InternalTester = (*Fs)(nil)